Subscriber stops acknowledging messages after running for a few minutes
See original GitHub issueI have sort of a weird issue and explaining it is somewhat difficult so please bear with me. It sounds very similar to an issue I read on the python pubsub github page (https://github.com/GoogleCloudPlatform/google-cloud-python/issues/4274).
I have a worker service on App Engine that I’m using to pull messages off of a PubSub topic, it is just listening to the topic and when it receives a message, it pushes it to an ElasticSearch instance that I have. I have set the minimum number of instances of the worker to a high number (20) to attempt to process a large number of records in the queue. The issue is that it seems on initial startup of the instances, they process and acknowledge a bunch of records but after a while stop acknowledging them (see the images below, the large spikes in the acknowledge graph seem to correspond to deployments/restarts of the worker instances).
I have attached my code, the subscription listener is invoked by calling the subscribe method in PubSubSerice.js in worker.js, I’ve attached the relevant portions of worker.js.
Any help is greatly appreciated!
// worker.js
pubSubService.subscribe('driver-log-sync', 'driver-log-sync', pubSubService.syncDriverLog)
pubSubService.subscribe('driver-log-sync-delete', 'driver-log-sync-delete', pubSubService.syncDriverLogDelete)
pubSubService.subscribe('hos-event-history-sync', 'hos-event-history-sync', pubSubService.syncHosEventHistory)
// PubSubService.js
const Pubsub = require('@google-cloud/pubsub')
const LoggingService = require('./loggingService')
const DriverLogsService = require('./driverLogsService')
const path = require('path')
const PQueue = require('p-queue')
let config = {
projectId: process.env.GCLOUD_PROJECT
}
const pubsub = Pubsub(config)
const queue = new PQueue({concurrency: 4})
function subscribe (topicName = '', subscriptionName = '', cb) {
function handleMessage (message) {
message.data = message.data.toString('utf-8')
cb(null, message)
}
function handleError (err) {
console.log('ERR')
LoggingService.error(err)
}
if (subscriptionName !== '' && topicName !== '') {
let topic = pubsub.topic(topicName)
let subscription = topic.subscription(subscriptionName)
subscription.on('message', handleMessage)
subscription.on('error', handleError)
LoggingService.info(`Listening to topic ${topicName} via subscription ${subscriptionName}`)
} else {
cb(new Error('Missing topic name and/or subscription name.'))
}
}
async function syncDriverLog (err, message) {
try {
if (err) {
throw new Error(err.message)
}
let { dotNumber, logDate, driverId } = message.attributes
if (!dotNumber) {
throw new Error('Missing DOT number for driver-log-sync')
}
if (!logDate) {
throw new Error('Missing Log Date for driver-log-sync')
}
if (!driverId) {
throw new Error('Missing Driver Id for driver-log-sync')
}
queue.add(async () => {
try {
await delay(25)
await DriverLogsService.syncDriverLogToElasticSearch(dotNumber, logDate, driverId, (new Date(message.publishTime).getTime()))
message.ack()
LoggingService.log(`Successfully synced log for driver: ${driverId}, dotNumber: ${dotNumber}, logDate: ${logDate}`)
} catch (err) {
message.ack()
LoggingService.error(`Error syncing log to ElasticSearch for driver: ${driverId}, dotNumber: ${dotNumber}, logDate: ${logDate}`, err)
}
})
} catch (err) {
message.ack()
LoggingService.error(`Error syncing log to ElasticSearch for message: ${message.attributes}`, err)
}
}
async function syncDriverLogDelete (err, message) {
try {
if (err) {
throw new Error(err.message)
}
let { dotNumber, logDate, driverId } = message.attributes
if (!dotNumber) {
throw new Error('Missing DOT number for driver-log-sync')
}
if (!logDate) {
throw new Error('Missing Log Date for driver-log-sync')
}
if (!driverId) {
throw new Error('Missing Driver Id for driver-log-sync')
}
queue.add(async () => {
try {
await delay(25)
await DriverLogsService.deleteDriverLogFromElasticSearch(dotNumber, logDate, driverId)
message.ack()
LoggingService.log(`Successfully deleted log for driver: ${driverId}, dotNumber: ${dotNumber}, logDate: ${logDate}`)
} catch (err) {
message.ack()
LoggingService.error(`Error deleting log for driver: ${driverId}, dotNumber: ${dotNumber}, logDate: ${logDate}`, err)
}
})
} catch (err) {
message.ack()
LoggingService.error(`Error syncing log to ElasticSearch for message: ${message.attributes}`, err)
}
}
async function syncHosEventHistory (err, message) {
try {
if (err) {
throw new Error(err.message)
}
let { hosEventHistoryNodeId, driverId } = message.attributes
let { hosEvent } = JSON.parse(message.data)
if (!hosEventHistoryNodeId) {
throw new Error('Missing HOS Event History Node Id for hos-event-history-sync')
}
if (!hosEvent) {
throw new Error('Missing HOS Event for hos-event-history-sync')
}
if (!driverId) {
throw new Error('Missing User Id for hos-event-history-sync')
}
queue.add(async () => {
try {
await delay(25)
await DriverLogsService.syncHosEventHistoryToElasticSearch(hosEventHistoryNodeId, hosEvent, driverId, (new Date(message.publishTime).getTime()))
message.ack()
LoggingService.log(`Successfully synced history for driver: ${driverId}, historyNodeId: ${hosEventHistoryNodeId}`)
} catch (err) {
message.ack()
LoggingService.error(`Error syncing log to ElasticSearch for message: ${message.attributes}`, err)
}
})
} catch (err) {
message.ack()
LoggingService.error('Error syncing log to ElasticSearch', err)
}
}
function delay (dur) {
return new Promise((resolve) => {
setTimeout(() => {
resolve()
}, dur)
})
}
module.exports = {
subscribe,
syncDriverLog,
syncDriverLogDelete,
syncHosEventHistory
}
Environment details
- OS: App Engine (Linux Debian Jessie)
- Node.js version: 8.4.0
- npm version: 5.3.0
- @google-cloud/pubsub version: 0.14.2
Issue Analytics
- State:
- Created 6 years ago
- Comments:6 (4 by maintainers)
Top GitHub Comments
@ctavan Thanks for your suggestion, it doesn’t look like my instances were getting over 40% CPU Utilization so I doubt that’s the problem.
@jonparrott I upgraded to the latest version of this package and it seems to be working as I expect now. Thanks for your help!
@benhunt29 thanks for the update! Please let us know if you start to experience any more issues.