findAndCountAll with virtual columns and "having", drops virtual column in the count query.

Following up on https://github.com/sequelize/sequelize/issues/9739, which you closed. It’s still an issue with latest sequelize 4.38

Using MySQL 5.7 Sequelize 4.38

Same error, still valid as above:

let options = {
   attributes: [
      'id',
      [sequelize.literal( `select 1` ), `has_access`]
   ],
   having: {'$has_access$': 1},
}

let result = await myTable.findAndCountAll(options);

gives error:

{
    "error": "error",
    "err": {
        "name": "SequelizeDatabaseError",
        "parent": {
            "code": "ER_BAD_FIELD_ERROR",
            "errno": 1054,
            "sqlState": "42S22",
            "sqlMessage": "Unknown column 'has_access' in 'having clause'",
            "sql": "SELECT count(*) AS `count`
             FROM `mytable` AS `mytable` HAVING `has_access` = 1;"
        },
        "original": {
            "code": "ER_BAD_FIELD_ERROR",
            "errno": 1054,
            "sqlState": "42S22",
            "sqlMessage": "Unknown column 'has_access' in 'having clause'",
            "sql": "SELECT count(*) AS `count` 
                FROM `mytable` AS `mytable` HAVING `has_access` = 1;"
        },
        "sql": "SELECT count(*) AS `count` 
              FROM `mytable` AS `mytable` HAVING `has_access` = 1;"
    },
    "err_msg": "Unknown column 'has_access' in 'having clause'",
    "err_stack": "SequelizeDatabaseError: Unknown column 'has_access' in 'having clause'\n    at 

And now, I get the same error with this alternate query:

let options = {
   attributes: [
      'id',
      [sequelize.literal( `select 1` ), `has_access`]
   ],
   having: {'$has_access$': 1},
}

  let [countResult,findResult] = await Promise.all([ mytable.count(options), mytable.findAll(options) ]);
  let result = {count: countResult, rows: findResult};

If findAndCountAll has options to enable a virtual column, it isn’t clear from the documentation for findAndCountAll nor in the documentation for count

Seems like a bug… or unsupported feature of using subqueries. Any ideas?

Have you considered using SQL_CALC_FOUND_ROWS and SELECT FOUND_ROWS() as an alternate way to count? Is there a way to use this MySQL feature? Perhaps I can generate a raw MySQL query with findAll(), and wrap with SQL_CALC_FOUND_ROWS and SELECT FOUND_ROWS()…

[alexbor]

Can this be re-opened? This is still an issue and means we can’t use findAllAndCount when we’re also using a virtual field/aggregator on the attributes.

[subatomicglue]

Here is some code that implements a full count needed for pagination, and supports virtual (calculated) columns with having as shown above:

// generate the MySQL query string,  without executing the query
// see https://github.com/sequelize/sequelize/issues/2325#issuecomment-366060303
function getSqlFromFindAll(Model, options) {
  const id = require('node-uuid').v4().replace( /-/g, '' );

  return new Promise((resolve, reject) => {
      Model.addHook('beforeFindAfterOptions', id, options => {
          Model.removeHook('beforeFindAfterOptions', id);

          resolve(Model.sequelize.dialect.QueryGenerator.selectQuery(Model.getTableName(), options, Model).slice(0, -1));

          return new Promise(() => {});
      });

      return Model.findAll(options).catch(reject);
  });
}

// THIS IS NOT USED, BUT INCLUDED FOR YOUR INTEREST...
// this mods the raw MySQL query to perform the query AND return the full count (needed for pagination)
// NOTE: limited usefulness without running the result through sequelize's object nesting logic... (returns raw rows)
// usage: let result = await sequelize.query( modQuery_runAndCount( query ) )
function modQuery_runAndCount( query ) {
  let q = query.replace( /^select/i, "select SQL_CALC_FOUND_ROWS" );
  q += ";\nSELECT FOUND_ROWS();";
  console.log( q );
  return q;
}

// this mods the raw MySQL query to return the full count (needed for pagination)
// usage: let count = (await sequelize.query( modQuery_count( query ) )).count
function modQuery_count( query ) {
  query = query.replace( /limit\s[0-9]+/i, "" );
  query = query.replace( /offset\s[0-9]+/i, "" );
  let uuid = require('node-uuid').v4().replace( /-/g, '' );
  let q = `select count(${uuid}.id) as count from (${query}) as ${uuid}`;
  console.log( q );
  return q;
}

// version of findAndCountAll that works with MySQL virtual columns as sequelize attributes
// (sequelize 4.38 and earlier drop all attributes in the count, removes virtual columns.)
async function findAndCountAll( model, options ) {
  let query = await getSqlFromFindAll( model, options );
  let [countResult,findResult] = await Promise.all([ sequelize.query( modQuery_count( query ) ), model.findAll(options) ]);
  let result = {count: countResult[0].count, rows: findResult};
  return result;
}

let result = await findAndCountAll( smOrgs, optionsObj );

outputs:

{
    "count": 34,
    "rows": [ {...}, ..., {...} ]
}

thank you to: https://github.com/sequelize/sequelize/issues/2325#issuecomment-366060303