Jobs getting stuck under small load

Created by: evanhuang8

I have been very interested in replacing kue with bull and I have been doing some testing, however, I keep running into the problem of jobs getting stuck.

Scripts

So my simply setup has two scripts, one is scheduler and the other one is worker

scheduler:

#!/usr/bin/env coffee

fs = require 'fs'
bull = require 'bull'
matador = require 'bull-ui/app'
co = require 'co'
Q = require 'q'

credentials = JSON.parse fs.readFileSync './credentials.json', 'utf8'

redisHost = credentials.redisHost
redisPort = credentials.redisPort
redisPass = credentials.redisPass
redisDBNumber = credentials.redisDBNumber

queue = bull 'test', redisPort, redisHost

app = matador
  redis:
    host: redisHost
    port: redisPort
    password: redisPass

sleep = (duration) ->
  deferred = Q.defer()
  setTimeout ->
    deferred.resolve()
    return
  , duration
  return deferred.promise

co ->

  for i in [0...100]
    job = yield queue.add {}, 
      delay: 10 * 1000
    console.log "Job #{job.jobId} scheduled"

  yield sleep 1000

  for i in [0...10]
    for j in [0...10]
      job = yield queue.add {}, 
        delay: 8 * 1000
      console.log "Job #{job.jobId} scheduled"
    yield sleep 1000

  return

.catch (err) ->

  console.log err

  return

Using a bit of ES6 generator logic here, so bare with me.

worker:

#!/usr/bin/env coffee

fs = require 'fs'
bull = require 'bull'
co = require 'co'
Q = require 'q'

credentials = JSON.parse fs.readFileSync './credentials.json', 'utf8'

redisHost = credentials.redisHost
redisPort = credentials.redisPort
redisPass = credentials.redisPass
redisDBNumber = credentials.redisDBNumber

queue = bull 'test', redisPort, redisHost

queue.process (job, done) ->
  console.log "Job #{job.jobId} is processed!"
  done()
  return

queue.on 'error', (err) ->
  console.log err
  return

Test runs

I am using iojs v2.5.0, tested with both redis 2.8.19 and 3.0.3. I flush redis before every run.

So when I run the scheduler first, and wait till all the jobs are scheduled, and then run the worker, then all jobs will go to completion.

If I remove the delay settings from the jobs, then everything seems fine - all jobs are processed.

When I run the worker and the scheduler at the same time - in other words, let the worker process as the scheduler goes, only a fraction of the jobs will be processed (ranging between 5 - 50 jobs out of the 200 jobs scheduled). Once the processing stops, no matter how long I wait, the jobs are stuck in the delayed state.

I suspect there is something going on with how the delayed jobs are setup, any insights?