fetchOffsets after resetOffsets returns -1 for all partitions

It seems fetchOffsets after resetOffsets keeps returning -1 to all partitions, even though there are previously stored offsets.

In principle, what I would like to do, is to output the current offsets to logs when a new consumer is added. In our use case, we should always start from the latest offset.

Here’s the code I used for testing (kafkajs version 1.4.4, kafka version 1.0.1):

const client = new kafka.Kafka({
    brokers: ['127.0.0.1:9092'],
    clientId: 'test-clientId'
});

const admin = client.admin();
await admin.connect();
await admin.resetOffsets({ groupId: 'test-groupId', topic: 'test-topic' });
// ALL PARTITION OFFSETS EQUAL -1
console.log(
    'FETCH OFFSETS',
    await admin.fetchOffsets({
        groupId: 'test-groupId',
        topic: 'test-topic'
    })
);

const consumer = client.consumer({
    groupId: 'test-groupId'
});
await consumer.subscribe({ topic: 'test-topic' });
await consumer.connect();
await consumer.run({
    eachMessage: async ({ message, partition }) => {
        console.log('RECEIVED MESSAGE', partition, message.offset);
    }
});

const producer = client.producer();
await producer.connect();
await producer.send({
    messages: [{ key: '', value: 'Hello world' }],
    topic: 'test-topic'
});
await producer.disconnect();
await consumer.disconnect();

await admin.resetOffsets({ groupId: 'test-groupId', topic: 'test-topic' });
// ALL PARTITION OFFSETS STILL EQUAL -1
console.log(
    'FETCH OFFSETS',
    await admin.fetchOffsets({
        groupId: 'test-groupId',
        topic: 'test-topic'
    })
);

await admin.disconnect();

If it helps, here’s the stdout after running the previous code block:

{"level":"INFO","timestamp":"2018-11-03T09:37:01.697Z","logger":"kafkajs","message":"[Consumer] Starting","groupId":"test-groupId"}
{"level":"INFO","timestamp":"2018-11-03T09:37:01.698Z","logger":"kafkajs","message":"[ConsumerGroup] Pausing fetching from 1 topics","topics":["test-topic"]}
{"level":"INFO","timestamp":"2018-11-03T09:37:01.712Z","logger":"kafkajs","message":"[Runner] Consumer has joined the group","groupId":"test-groupId","memberId":"test-clientId-70e47b46-abb4-4ce9-bc69-7b60c289d4ee","leaderId":"test-clientId-70e47b46-abb4-4ce9-bc69-7b60c289d4ee","isLeader":true,"memberAssignment":{"test-topic":[2,1,0]},"duration":14}
{"level":"INFO","timestamp":"2018-11-03T09:37:07.748Z","logger":"kafkajs","message":"[Consumer] Stopped","groupId":"test-groupId"}
FETCH OFFSETS [ { partition: 0, offset: '-1' },
  { partition: 1, offset: '-1' },
  { partition: 2, offset: '-1' } ]
{"level":"INFO","timestamp":"2018-11-03T09:37:07.759Z","logger":"kafkajs","message":"[Consumer] Starting","groupId":"test-groupId"}
{"level":"INFO","timestamp":"2018-11-03T09:37:07.767Z","logger":"kafkajs","message":"[Runner] Consumer has joined the group","groupId":"test-groupId","memberId":"test-clientId-91a4b8ee-da97-49c8-afae-c0db0c71f70e","leaderId":"test-clientId-91a4b8ee-da97-49c8-afae-c0db0c71f70e","isLeader":true,"memberAssignment":{"test-topic":[2,1,0]},"duration":8}
RECEIVED MESSAGE 0 15
{"level":"INFO","timestamp":"2018-11-03T09:37:08.821Z","logger":"kafkajs","message":"[Consumer] Stopped","groupId":"test-groupId"}
{"level":"INFO","timestamp":"2018-11-03T09:37:08.824Z","logger":"kafkajs","message":"[Consumer] Starting","groupId":"test-groupId"}
{"level":"INFO","timestamp":"2018-11-03T09:37:08.824Z","logger":"kafkajs","message":"[ConsumerGroup] Pausing fetching from 1 topics","topics":["test-topic"]}
{"level":"INFO","timestamp":"2018-11-03T09:37:08.830Z","logger":"kafkajs","message":"[Runner] Consumer has joined the group","groupId":"test-groupId","memberId":"test-clientId-d2bd49d8-31f4-4561-af03-41adabf9be62","leaderId":"test-clientId-d2bd49d8-31f4-4561-af03-41adabf9be62","isLeader":true,"memberAssignment":{"test-topic":[2,1,0]},"duration":6}
{"level":"INFO","timestamp":"2018-11-03T09:37:14.846Z","logger":"kafkajs","message":"[Consumer] Stopped","groupId":"test-groupId"}
FETCH OFFSETS [ { partition: 0, offset: '-1' },
  { partition: 1, offset: '-1' },
  { partition: 2, offset: '-1' } ]

[tulios]

@kpala issue #205 has your feature request, I’m closing this one for now. Thanks again

[tulios]

@kpala I think you are not giving enough time for your consumers to process the message, you are sending the message and disconnecting right after it. Another thing, the only way you have to know when the consumer has joined the group is listening to the instrumentation event GROUP_JOIN, something like:

const { GROUP_JOIN } = consumer.events
consumer.run({ ... }
// ...
consumer.on(GROUP_JOIN, async event => {
  console.log(e.payload. memberAssignment)
})