[Service Bus] Simultaneously call receiveMessages in queue receiver in peekLock mode will cause message lost(lock).

• Package Name: @azure/service-bus
• Package Version: 7.0.0-preview.8
• Operating system: Linux centos
• nodejs
  • version: 14.15.1
• browser
  • name/version:
• typescript
  • version:
• Is the bug related to documentation in
  • README.md
  • source code documentation
  • SDK API docs on https://docs.microsoft.com

Describe the bug Simultaneously call receiveMessages in queue receiver in peekLock mode will cause message lost(lock).

To Reproduce package.json

{
  "name": "azure_sbtest",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1"
  },
  "author": "",
  "license": "ISC",
  "dependencies": {
    "@azure/abort-controller": "^1.0.1",
    "@azure/service-bus": "^7.0.0-preview.8",
    "async-mutex": "^0.2.6"
  }
}

ext-store.js

const { ServiceBusClient, ServiceBusAdministrationClient } = require("@azure/service-bus");
//const { AzureLogger, setLogLevel } = require("@azure/logger");
const { AbortController } = require("@azure/abort-controller");

//setLogLevel("error");

class _mutaxmock {
  constructor(){}
  acquire() {return Promise.resolve(()=>{})}
  isLocked() {return false;}
}

//const Mutex = require('async-mutex').Mutex;
const Mutex = _mutaxmock;

class QueueServiceBus {
  constructor(config, logger) {
    this.config = config;
    this.logger = logger || { debug: ()=>{}, info: ()=>{}, error: ()=>{}};
    let _config = JSON.parse(JSON.stringify(config));
    this.serviceBusClient = new ServiceBusClient(this.config.connectionString);

    this.queuePrefix = _config.queuePrefix;
    this.provider = 'azure';
    this.mutex = new Mutex();
  }

  async receiveMessage (queueName, num, pullDuration, options) {
    if(this.mutex.isLocked()) {
      await new Promise(resolve => {setTimeout(() => {resolve();}, pullDuration * 1000);});
      return [];
    }
    const release = await this.mutex.acquire();
    try {
      let queueReceiver =  this.serviceBusClient.createReceiver(`${this.queuePrefix}-${queueName}`, { maxAutoLockRenewalDurationInMs: 0});
      let msgs = await queueReceiver.receiveMessages(parseInt(num), {maxWaitTimeInMs: pullDuration * 1000});
      await queueReceiver.close();
      return msgs;
    } finally {
      release();
    }
  }

  async deleteMessage (queueName, msg) {
    const release = await this.mutex.acquire();
    try{
      this.logger.debug(`azure delMessage ${msg} from ${queueName}`);
      let queueReceiver = this.serviceBusClient.createReceiver(`${this.queuePrefix}-${queueName}`, { maxAutoLockRenewalDurationInMs: 0});
      await queueReceiver.completeMessage(msg);
      await queueReceiver.close();
    }catch(error){
      this.logger.info(error);
    } finally {
      release();
    }
  }
}

Queue = {
  azure: QueueServiceBus
};

let _queue = null;

module.exports.getQueue = (config, logger) => {
  if (_queue) {
    return _queue;
  } else {
    const QueueImpl = Queue[config.provider.queue];
    _queue = new QueueImpl(config.queue[config.provider.queue], logger);
    return _queue
  }

};

testqueue.js

const config = {
  "provider": {
    "queue": "azure"
  },
  "queue": {
    "azure" : {
      "connectionString": "YOUR_CONN_STR",
      "queuePrefix" : "ddc-test-monitor"
    }
  },
};

const numOfFuncs = 10;

const queue = require('./ext-store').getQueue(config);

function getRandomInt(max) {return Math.floor(Math.random() * Math.floor(max));}

sleep = async (ts) => {
  return new Promise(resolve => {
    setTimeout(() => {
      resolve(true);
    }, ts || 2000);
  });
};

let sum = 0;
async function main(tn) {
  console.log(`enter task::${tn}`);
  while (true){
    let messages = await queue.receiveMessage(`preFilter-windowsFile2`, getRandomInt(10)+1, 5);
    if (messages.length) {
      sum += messages.length;
      console.log(`${new Date().toISOString()} task::${tn}  received::${messages.length}, current sum:: ${sum}`);
    }
    for(let message of messages){
      await queue.deleteMessage('preFilter-windowsFile2', message)
    }
    await sleep(2000)
  }

}
for (let idx=0; idx < numOfFuncs ; idx++){
  main(idx);
}

Steps to reproduce the behavior:

1. we have a thin wrapper around SDK for multi-platform support while we think is simple enough.
2. when we send 40 messages to queue in one time, and use node testqueue.js to receive, some messages are missing and from azure servicebus portal, the missing message seems locked.

[root@ip-10-180-12-91 testqueue]# node testqueue.js
enter task::0
enter task::1
enter task::2
enter task::3
enter task::4
enter task::5
enter task::6
enter task::7
enter task::8
enter task::9
2020-11-30T08:43:39.103Z task::0  receive::7 msg, current sum:: 7
2020-11-30T08:43:39.685Z task::2  receive::1 msg, current sum:: 8
2020-11-30T08:43:39.803Z task::1  receive::5 msg, current sum:: 13
2020-11-30T08:43:40.667Z task::3  receive::2 msg, current sum:: 15
2020-11-30T08:43:40.934Z task::4  receive::2 msg, current sum:: 17
2020-11-30T08:43:41.017Z task::7  receive::1 msg, current sum:: 18
2020-11-30T08:43:41.123Z task::5  receive::1 msg, current sum:: 19
2020-11-30T08:43:41.322Z task::6  receive::1 msg, current sum:: 20
2020-11-30T08:43:41.835Z task::8  receive::2 msg, current sum:: 22
2020-11-30T08:43:42.018Z task::9  receive::2 msg, current sum:: 24
2020-11-30T08:43:43.412Z task::2  receive::1 msg, current sum:: 25
2020-11-30T08:43:44.313Z task::3  receive::2 msg, current sum:: 27
2020-11-30T08:43:45.486Z task::1  receive::6 msg, current sum:: 33
... missing or hang from here...

1. If we change the variable numOfFuncs in testqueue.js to 1, then we can receive the whole messages.
2. If we introduce const Mutex = require('async-mutex').Mutex; in ext-store.js line 13, then we can also receive the whole messages.
3. As we can workaround by adding mutex or reduce the function number, we suspect there may be sth wrong when receiveMessages or createReceiver got reentry.

Expected behavior We except no message lost(lock) when there are multiple simultaneously calls to sdk. we impl a web server and recv message as per incoming request, so there will be reentry case.

Screenshots If applicable, add screenshots to help explain your problem.

Additional context azure_sbtest.zip

[enoeden]

hi @richardpark-msft

After my brief trial, seems the fix works and thanks a lot for the quick response and help!

[HarshaNalluru]

@enoeden Thanks for the confirmation.

Version 7.0.2 for @azure/service-bus has been published and it should contain the fix. 😃

You can install it with npm install @azure/service-bus.

Feel free to open new issues in this repository in case you run into problems.