Messages not received after a drain, if credit was not added immediately

Below is the code sample using rhea-promise that shows the problem we are seeing. Please ensure that the queue that is being used is empty before each attempt to run the sample

const { Connection, ReceiverEvents, delay } = require("rhea-promise");

const host = "";
const username = "";
const password = "";
const port = 5671;
const receiverAddress = "";
const senderAddress = "";

async function receiveOne(receiver) {
  return new Promise(resolve => {
    const messageWaitTimer = setTimeout(() => {
      console.log("Didnt receive any message in 5 seconds");

      clearTimeout(messageWaitTimer);
      receiver.removeListener(ReceiverEvents.message, onReceiveMessage);

      if (receiver.credit > 0) {
        console.log("Draining leftover credits");
        receiver.drain = true;
        receiver.addCredit(1);
      } else {
        receiver.removeListener(ReceiverEvents.receiverDrained, onReceiveDrain);
        resolve();
      }
    }, 5000);

    const onReceiveMessage = async context => {
      console.log("Received message: %O", context.message.body);

      clearTimeout(messageWaitTimer);
      receiver.removeListener(ReceiverEvents.message, onReceiveMessage);
      receiver.removeListener(ReceiverEvents.receiverDrained, onReceiveDrain);

      resolve();
    };

    const onReceiveDrain = () => {
      console.log("Drain done");
      receiver.removeListener(ReceiverEvents.receiverDrained, onReceiveDrain);
      receiver.drain = false;
      resolve();
    };

    console.log("Adding 1 credit");
    receiver.addCredit(1);
    receiver.on(ReceiverEvents.message, onReceiveMessage);
    receiver.on(ReceiverEvents.receiverDrained, onReceiveDrain);
  });
}

async function main() {
  const connection = new Connection({
    transport: "tls",
    host: host,
    hostname: host,
    username: username,
    password: password,
    port: port,
    reconnect: false
  });
  await connection.open();

  const receiver = await connection.createReceiver({
    name: "receiver-1",
    source: {
      address: receiverAddress
    },
    credit_window: 0
  });

  const sender = await connection.createSender({
    name: "sender-1",
    target: {
      address: senderAddress
    }
  });

  // This will receive 0 msgs as the queue is empty when we start
  await receiveOne(receiver);

  console.log("Sending message");
  await sender.send({ body: "Hello World" });
  console.log("Sent message");

  // Without this delay, we receive the sent message in the next `receiveOne` call
  await delay(1000);
  console.log("Done sleeping");

  await receiveOne(receiver);

  console.log("Closing");
  await sender.close();
  await receiver.close();
  await connection.close();
}

main().catch(err => console.log(err));

Some details about the code:

The receiveOne call

• attaches the event handlers for the message and drain events,
• adds a credit
• returns a promise.
• the promise is resolved if either a message is received or 5 seconds have passed.
• In the latter case, credits are drained. The handlers are removed before resolving the promise.

The sequence of events are

1. receiveOne is called, expected to get no messages, and therefore credits are drained
2. a message is sent
3. we wait for 1 sec
4. receiveOne is called again, expected to get the message, but no message is received.

Remove step 3 from above, and we receive the expected message in the last step.

From the logs, the difference I see in the 2 cases is the point at which the credits get added for the second receiveOne call. If I move the adding of credit out of receiveOne and right before the delay (right after the send), then I can see that rhea indeed receives the message

Relevant logs with DEBUG=rhea*,-rhea:io,-rhea:raw

With the delay:

Adding 1 credit
  rhea:frames [connection-1]:0 -> flow#13 {"incoming_window":2048,"outgoing_window":4294967295,"link_credit":1}  +3ms
Didnt receive any message in 5 seconds
Draining leftover credits
  rhea:frames [connection-1]:0 -> flow#13 {"incoming_window":2048,"outgoing_window":4294967295,"link_credit":2,"drain":true}  +5s
  rhea:frames [connection-1]:0 <- flow#13 {"incoming_window":5000,"next_outgoing_id":1,"outgoing_window":2047,"delivery_count":2,"drain":true}  +25ms
  rhea:events [connection-1] Link got event: receiver_flow +5s
  rhea-promise:receiver [connection-1] receiver got event: 'receiver_flow'. Re-emitting the translated context. +5s
  rhea-promise:translate [connection-1] Translating the context for event: 'receiver_flow'. +5s
  rhea:events [connection-1] Link got event: receiver_drained +0ms
  rhea-promise:receiver [connection-1] receiver got event: 'receiver_drained'. Re-emitting the translated context. +0ms
  rhea-promise:translate [connection-1] Translating the context for event: 'receiver_drained'. +0ms
Drain done
Sending message
  rhea:message Encoding section 1 of 3: Typed { type: TypeDesc { name: 'List0', typecode: 69, width: 0, category: 1, create: { [Function] typecode: 69 } }, value: [], descriptor: { [Number: 112] type: TypeDesc { name: 'SmallUlong', typecode: 83, width: 1, category: 1, read: [Function: read], write: [Function: write], create: [Object] }, value: 112 } } +0ms
  rhea:message Encoding section 2 of 3: Typed { type: TypeDesc { name: 'List0', typecode: 69, width: 0, category: 1, create: { [Function] typecode: 69 } }, value: [], descriptor: { [Number: 115] type: TypeDesc { name: 'SmallUlong', typecode: 83, width: 1, category: 1, read: [Function: read], write: [Function: write], create: [Object] }, value: 115 } } +1ms
  rhea:message Encoding section 3 of 3: Typed { type: TypeDesc { name: 'Str8', typecode: 161, width: 1, category: 2, encoding: 'utf8', create: { [Function] typecode: 161 } }, value: 'Hello World', descriptor: { [Number: 119] type: TypeDesc { name: 'SmallUlong', typecode: 83, width: 1, category: 1, read: [Function: read], write: [Function: write], create: [Object] }, value: 119 } } +0ms
  rhea:message encoded 24 bytes +1ms
Sent message
  rhea:frames [connection-1]:1 -> transfer#14 {"delivery_tag":{"type":"Buffer","data":[48]}} <Buffer 00 53 70 45 00 53 73 45 00 53 77 a1 0b 48 65 6c 6c 6f 20 57 6f 72 6c 64> +5ms
  rhea:frames [connection-1]:1 <- disposition#15 {"role":true,"settled":true,"state":[]}  +57ms
  rhea:events [connection-1] Received disposition for outgoing transfers +62ms
  rhea:events [connection-1] Link got event: accepted +0ms
  rhea-promise:sender [connection-1] sender got event: 'accepted'. Re-emitting the translated context. +5s
  rhea-promise:translate [connection-1] Translating the context for event: 'accepted'. +63ms
  rhea:events [connection-1] Link got event: settled +1ms
  rhea-promise:sender [connection-1] sender got event: 'settled'. Re-emitting the translated context. +1ms
  rhea-promise:translate [connection-1] Translating the context for event: 'settled'. +0ms
Done sleeping
Adding 1 credit
  rhea:frames [connection-1]:0 -> flow#13 {"incoming_window":2048,"outgoing_window":4294967295,"link_credit":1}  +943ms
Didnt receive any message in 5 seconds
Draining leftover credits
  rhea:frames [connection-1]:0 -> flow#13 {"incoming_window":2048,"outgoing_window":4294967295,"link_credit":2,"drain":true}  +5s
  rhea:frames [connection-1]:0 <- flow#13 {"incoming_window":5000,"next_outgoing_id":1,"outgoing_window":2047,"delivery_count":2,"drain":true}  +25ms
  rhea:events [connection-1] Link got event: receiver_flow +6s
  rhea-promise:receiver [connection-1] receiver got event: 'receiver_flow'. Re-emitting the translated context. +6s
  rhea-promise:translate [connection-1] Translating the context for event: 'receiver_flow'. +6s
  rhea:events [connection-1] Link got event: receiver_drained +0ms
  rhea-promise:receiver [connection-1] receiver got event: 'receiver_drained'. Re-emitting the translated context. +0ms
  rhea-promise:translate [connection-1] Translating the context for event: 'receiver_drained'. +0ms
Drain done
Closing

Logs for the same without the delay i.e the line await delay(1000); commented:

Adding 1 credit
  rhea:frames [connection-1]:0 -> flow#13 {"incoming_window":2048,"outgoing_window":4294967295,"link_credit":1}  +2ms
Didnt receive any message in 5 seconds
Draining leftover credits
  rhea:frames [connection-1]:0 -> flow#13 {"incoming_window":2048,"outgoing_window":4294967295,"link_credit":2,"drain":true}  +5s
  rhea:frames [connection-1]:0 <- flow#13 {"incoming_window":5000,"next_outgoing_id":1,"outgoing_window":2047,"delivery_count":2,"drain":true}  +22ms
  rhea:events [connection-1] Link got event: receiver_flow +5s
  rhea-promise:receiver [connection-1] receiver got event: 'receiver_flow'. Re-emitting the translated context. +5s
  rhea-promise:translate [connection-1] Translating the context for event: 'receiver_flow'. +5s
  rhea:events [connection-1] Link got event: receiver_drained +0ms
  rhea-promise:receiver [connection-1] receiver got event: 'receiver_drained'. Re-emitting the translated context. +0ms
  rhea-promise:translate [connection-1] Translating the context for event: 'receiver_drained'. +0ms
Drain done
Sending message
  rhea:message Encoding section 1 of 3: Typed { type: TypeDesc { name: 'List0', typecode: 69, width: 0, category: 1, create: { [Function] typecode: 69 } }, value: [], descriptor: { [Number: 112] type: TypeDesc { name: 'SmallUlong', typecode: 83, width: 1, category: 1, read: [Function: read], write: [Function: write], create: [Object] }, value: 112 } } +0ms
  rhea:message Encoding section 2 of 3: Typed { type: TypeDesc { name: 'List0', typecode: 69, width: 0, category: 1, create: { [Function] typecode: 69 } }, value: [], descriptor: { [Number: 115] type: TypeDesc { name: 'SmallUlong', typecode: 83, width: 1, category: 1, read: [Function: read], write: [Function: write], create: [Object] }, value: 115 } } +1ms
  rhea:message Encoding section 3 of 3: Typed { type: TypeDesc { name: 'Str8', typecode: 161, width: 1, category: 2, encoding: 'utf8', create: { [Function] typecode: 161 } }, value: 'Hello World', descriptor: { [Number: 119] type: TypeDesc { name: 'SmallUlong', typecode: 83, width: 1, category: 1, read: [Function: read], write: [Function: write], create: [Object] }, value: 119 } } +0ms
  rhea:message encoded 24 bytes +1ms
Sent message
Done sleeping
Adding 1 credit
  rhea:frames [connection-1]:0 -> flow#13 {"incoming_window":2048,"outgoing_window":4294967295,"link_credit":1}  +4ms
  rhea:frames [connection-1]:1 -> transfer#14 {"delivery_tag":{"type":"Buffer","data":[48]}} <Buffer 00 53 70 45 00 53 73 45 00 53 77 a1 0b 48 65 6c 6c 6f 20 57 6f 72 6c 64> +1ms
  rhea:frames [connection-1]:1 <- disposition#15 {"role":true,"settled":true,"state":[]}  +124ms
  rhea:events [connection-1] Received disposition for outgoing transfers +128ms
  rhea:events [connection-1] Link got event: accepted +1ms
  rhea-promise:sender [connection-1] sender got event: 'accepted'. Re-emitting the translated context. +5s
  rhea-promise:translate [connection-1] Translating the context for event: 'accepted'. +129ms
  rhea:events [connection-1] Link got event: settled +0ms
  rhea-promise:sender [connection-1] sender got event: 'settled'. Re-emitting the translated context. +0ms
  rhea-promise:translate [connection-1] Translating the context for event: 'settled'. +0ms
  rhea:frames [connection-1]:0 <- transfer#14 {"delivery_tag":{"type":"Buffer","data":[214,206,155,106,25,164,186,64,184,124,124,23,61,250,0,46]},"batchable":true} <Buffer 00 53 70 c0 0a 05 40 40 70 48 19 08 00 40 43 00 53 71 c1 24 02 a3 10 78 2d 6f 70 74 2d 6c 6f 63 6b 2d 74 6f 6b 65 6e 98 6a 9b ce d6 a4 19 40 ba b8 7c ... > +21ms
  rhea:message decoding section: Typed { type: TypeDesc { name: 'List8', typecode: 192, width: 1, category: 3, create: { [Function] typecode: 192 } }, value: [ Typed { type: [Object], value: null }, Typed { type: [Object], value: null }, { [Number: 1209600000] type: [Object], value: 1209600000 }, Typed { type: [Object], value: null }, { [Number: 0] type: [Object], value: 0 } ], descriptor: { [Number: 112] type: TypeDesc { name: 'SmallUlong', typecode: 83, width: 1, category: 1, read: [Function: read], write: [Function: write], create: [Object] }, value: 112 } } of type: { [Number: 112] type: TypeDesc { name: 'SmallUlong', typecode: 83, width: 1, category: 1, read: [Function: read], write: [Function: write], create: { [Function] typecode: 83 } }, value: 112 } +148ms
  rhea:message decoding section: Typed { type: TypeDesc { name: 'Map8', typecode: 193, width: 1, category: 3, create: { [Function] typecode: 193 } }, value: [ Typed { type: [Object], value: 'x-opt-lock-token' }, Typed { type: [Object], value: <Buffer 6a 9b ce d6 a4 19 40 ba b8 7c 7c 17 3d fa 00 2e> } ], descriptor: { [Number: 113] type: TypeDesc { name: 'SmallUlong', typecode: 83, width: 1, category: 1, read: [Function: read], write: [Function: write], create: [Object] }, value: 113 } } of type: { [Number: 113] type: TypeDesc { name: 'SmallUlong', typecode: 83, width: 1, category: 1, read: [Function: read], write: [Function: write], create: { [Function] typecode: 83 } }, value: 113 } +1ms
  rhea:message decoding section: Typed { type: TypeDesc { name: 'Map8', typecode: 193, width: 1, category: 3, create: { [Function] typecode: 193 } }, value: [ Typed { type: [Object], value: 'x-opt-enqueued-time' }, { [Number: 1551840769002] type: [Object], value: 1551840769002 }, Typed { type: [Object], value: 'x-opt-sequence-number' }, Typed { type: [Object], value: <Buffer 00 2e 00 00 00 00 00 1e> }, Typed { type: [Object], value: 'x-opt-enqueue-sequence-number' }, { [Number: 0] type: [Object], value: 0 }, Typed { type: [Object], value: 'x-opt-partition-id' }, { [Number: 46] type: [Object], value: 46 }, Typed { type: [Object], value: 'x-opt-partition-key' }, Typed { type: [Object], value: '287' }, Typed { type: [Object], value: 'x-opt-locked-until' }, { [Number: 1551840799049] type: [Object], value: 1551840799049 } ], descriptor: { [Number: 114] type: TypeDesc { name: 'SmallUlong', typecode: 83, width: 1, category: 1, read: [Function: read], write: [Function: write], create: [Object] }, value: 114 } } of type: { [Number: 114] type: TypeDesc { name: 'SmallUlong', typecode: 83, width: 1, category: 1, read: [Function: read], write: [Function: write], create: { [Function] typecode: 83 } }, value: 114 } +1ms
  rhea:message decoding section: Typed { type: TypeDesc { name: 'List8', typecode: 192, width: 1, category: 3, create: { [Function] typecode: 192 } }, value: [ Typed { type: [Object], value: '9fc52544dd0e4bc7a83424c6e7b470ff' }, Typed { type: [Object], value: null }, Typed { type: [Object], value: null }, Typed { type: [Object], value: null }, Typed { type: [Object], value: null }, Typed { type: [Object], value: null }, Typed { type: [Object], value: null }, Typed { type: [Object], value: null }, Typed { type: [Object], value: null }, Typed { type: [Object], value: null }, Typed { type: [Object], value: null }, Typed { type: [Object], value: null }, Typed { type: [Object], value: null } ], descriptor: { [Number: 115] type: TypeDesc { name: 'SmallUlong', typecode: 83, width: 1, category: 1, read: [Function: read], write: [Function: write], create: [Object] }, value: 115 } } of type: { [Number: 115] type: TypeDesc { name: 'SmallUlong', typecode: 83, width: 1, category: 1, read: [Function: read], write: [Function: write], create: { [Function] typecode: 83 } }, value: 115 } +1ms
  rhea:message decoding section: Typed { type: TypeDesc { name: 'Str8', typecode: 161, width: 1, category: 2, encoding: 'utf8', create: { [Function] typecode: 161 } }, value: 'Hello World', descriptor: { [Number: 119] type: TypeDesc { name: 'SmallUlong', typecode: 83, width: 1, category: 1, read: [Function: read], write: [Function: write], create: [Object] }, value: 119 } } of type: { [Number: 119] type: TypeDesc { name: 'SmallUlong', typecode: 83, width: 1, category: 1, read: [Function: read], write: [Function: write], create: { [Function] typecode: 83 } }, value: 119 } +1ms
  rhea:events [connection-1] Link got event: message +26ms
  rhea-promise:receiver [connection-1] receiver got event: 'message'. Re-emitting the translated context. +155ms
  rhea-promise:translate [connection-1] Translating the context for event: 'message'. +27ms
Received message: 'Hello World'
  rhea:frames [connection-1]:0 -> disposition#15 {"role":true,"settled":true,"state":[]}  +7ms
Closing

[grs]

1.0.0 is identical to 0.3.11 expect that the rpc module has been removed. The reason to change was that apparently npm doesn’t handle semantic versioning fully with pre 1.0.0 versions meaning you can’t fix it to a particular minor release and get all bug fixes. I was asked to change in an issue not so long ago and have been postponing but decided just to do it now.

[grs]

The link credit is always relative to the stated delivery-count, i.e. in effect it says from message N you can send M more where N is delivery-id and M is link credit. The delivery count is an implicit counter, incremented on every message. If a link is drained and there were no messages to be sent (or not enough for all the credit), the delivery-count is moved on explicitly by the sender. Prior to this fix rhea was not handling this correctly, so after a drain where the messages available were less than credit would allow the delivery-count got out of sequence between the receiver (rhea) and the sender. Subsequent attempts to grant credit were therefore considered stale (e.g. saying you can have one more credit from message 1, where the sender had already moved the counter to 2). Hope that makes some sense.