Same job.id being processed by multiple cluster workers.

When using cluster to process jobs, I am seeing this issue intermittently. The same job is processed simultaneously by 2 or 3 different workers.

I have attached an image showing the job log printing out multiple pids and duplicate line items as the job is processed twice.

My code has the following directory structure: /jobs —> sendEmailToGroup.js —> sendEmail.js server.js

server.js

var debug = require('debug')('cluster')
, cluster = require('cluster')
, config = require('./config.js');


//--------------------------------------------------------------------------------------------
// cluster
//--------------------------------------------------------------------------------------------

if (cluster.isMaster) {
  var clusterWorkerSize = 4; // set this manually. I am running on a Quadcore Mac mini
  console.log("master pid %s", process.pid);
  for (var i = 0; i < clusterWorkerSize; i++) {
    cluster.fork();
  }
} 
else {
process._debugPort = 5858 + cluster.worker.id;
debug("worker %s pid %s debugPort %s", cluster.worker && cluster.worker.id, process.pid, process._debugPort);

// I run the server only on worker processes. It is my understanding that these worker processes share nothing, and that I should be able to create a kue server on each worker. 

// I am wrapping Kue in express for future feature additions to this system.

//--------------------------------------------------------------------------------------------
// Express
//--------------------------------------------------------------------------------------------
var express = require('express');
var app = express();

app.set('port', config.PORT);

//--------------------------------------------------------------------------------------------
// Kue
//--------------------------------------------------------------------------------------------
var kue = require('kue');
var jobs = kue.createQueue({
    prefix: config.KUE_TYPE
    , redis: {
        port: config.REDIS_PORT || 6379
      , host: config.REDIS_HOST || '127.0.0.1'
    }
});

// make sure each worker promotes jobs
// if this is my issue...I'm still confused as to how it would breaks things
  var promote_interval = 100;
  jobs.promote(promote_interval);

  jobs.process('sendEmailToGroup', 1, require('./jobs/sendEmailToGroup'));
  jobs.process('sendEmail', 20, require('./jobs/sendEmail'));


 // simple express middleware
  app.use(function(req, res, next){
  kue.app.set('title', req.host);
    next();
  });

// 
  app.use(kue.app);
  app.listen(app.get('port'));

}

sendEmailToGroup.js

var cluser = require('cluster')
, config = require('../config.js')
, dpd = require('../dpd-init.js')(config.DPD_ROOT)
, _ = require('lodash')
, moment = require('moment')
, kue = require('kue');

var jobs = kue.createQueue({
    prefix: config.KUE_TYPE
    , redis: {
        port: config.REDIS_PORT || 6379
      , host: config.REDIS_HOST || '127.0.0.1'
    }
});

require('../parse-init.js');

exports = module.exports = (function(job, done){
  var jobKey = "job_" + job.id;
  job.log('start ' + jobKey + ' pid ' + process.pid);
  var queryCompletion = 0;
  var limit = 100;
  var upperLimit = 10001;
  var completion = upperLimit + limit;
  var skip = 0;
  var lastResultCount = limit;
  var totalResultCount = 0;

  var lastSent = moment().toDate();

  if(job.data.lastSent) {
    lastSent = moment(job.data.lastSent).toDate();
  }
  var query =  new Parse.Query(User);
  query.limit(limit);
  query.equalTo('emailGroup', job.data.emailGroup); 
  query.exists("email"); 
  query.notEqualTo('doNotSendEmail', true); 
  query.descending("createdAt");
  promiseWhile(function() {
    if(lastResultCount >= limit && lastResultCount != 0 && skip < upperLimit){
      return true;
    }
    else {
      return false;
    }
  },
  function() {

    query.lessThan("createdAt", lastSent);
    var mainPromise = new Parse.Promise();
    mainPromise = query.find().then(function(results){

      lastResultCount = results.length;
      totalResultCount += lastResultCount;
      skip += limit;

      if (lastResultCount < limit && upperLimit > totalResultCount){
          completion = totalResultCount;
      }
      job.progress(queryCompletion, completion);

      var promises = [];
      _.each(results, function(user) {
        var promise = new Parse.Promise();
        promises.push(promise);
        if(user.get('email') !== undefined){
          var userId = user.id;
          var userCreatedAt = user.createdAt;
          lastSent = moment(userCreatedAt).toDate();
          var toName = user.get('kFullNameKey') || user.get('username');
          var toEmail = user.get('email');
          var emailGroup = user.get('emailGroup');
          var sendEmail = jobs.create('sendEmail', {
            title: '' + job.data.emailType + ': ' + job.data.typeObjectId,
            emailType: job.data.emailType,
            typeObjectId: job.data.typeObjectId,
            template: job.data.template,
            userId: userId,
            toName: toName,
            toEmail: toEmail,
            fromName: job.data.fromName,
            fromEmail: job.data.fromEmail,
            subject: job.data.subject,
            emailGroup: emailGroup,
            emailScheduleId: job.data.emailScheduleId,
            userCreatedAt: userCreatedAt,
            user: user
          }).attempts(2).save();
          promise.resolve();
          return promise;
        }
      });
    job.log("Retrieved " + results.length + " Users" + " / " + totalResultCount + " Total");

    return Parse.Promise.when(promises);
    }, function(error) {
      //fail the job on error
      job.log("" + error.message);
      done(error);
  });
  return mainPromise;
  }).then(function() {
      job.log('job done');
      done();
  }, function(error) {

  });
});

Any ideas (other than “don’t use Parse”) would be extremely helpful.

[victornikitin]

Same problem. Several cluster workers get the same message.

Code (simplified):

if (cluster.isMaster) {
  for (let i = 0; i < 10; i++) {
    cluster.fork();
  }
} else {
  const queue = kue.createQueue();
  
  queue.process("q", 1, (job, done) => {
       // .. code here
      done()
    });
}

Is threre any solution?

[victornikitin]

@kimballfrank have you found the solution?