deadlock when using `batch` with `flatMap` and mongo stream as source (node >= 10)
See original GitHub issueThe following test deadlocks (toCallback
never gets called):
/* eslint-disable no-unused-vars, no-shadow, no-redeclare */
var _, EventEmitter = require('events').EventEmitter,
through = require('through'),
// sinon = require('sinon'),
Stream = require('stream'),
streamify = require('stream-array'),
concat = require('concat-stream'),
// Promise = RSVP.Promise,
transducers = require('transducers-js'),
bluebird = require('bluebird'),
runTask = require('orchestrator/lib/runTask'),
fl = require('fantasy-land'),
bufferFrom = require('buffer-from');
if (global.highland != null) {
_ = global.highland;
}
else {
_ = require('../lib/index');
}
// Use bluebird cancellation. We want to test against it.
bluebird.config({
cancellation: true,
longStackTraces : true
});
var MongoClient = require('mongodb').MongoClient
exports['batch - mongodb'] = function (test) {
test.expect(1);
MongoClient.connect('mongodb://localhost:27017')
.then(async (client) => {
var db = client.db("test")
var col = db.collection("test")
const docs = Array.from(Array(22).keys()).map( _id => { return { _id } })
await col.removeMany({})
await col.insertMany(docs)
cursor = col.find()
cursor.batchSize(2)
// Changing batch size to 3 causes the test to pass.
// cursor.batchSize(3)
_(cursor.stream())
.batch(11)
// Changing batch size to 12 causes the test to pass.
// .batch(12)
.flatMap( x => {
console.log("x:", x)
return _(x)
})
// If flatMap above is replaced with .map, the test passes.
// .map( x => {
// console.log("x:", x)
// return x
// })
.reduce(
(acc, x) => {
console.log("conc", acc, x)
return acc.concat(x)
},
[]
)
.toCallback((e, xs) => {
console.log("xs",xs)
test.same(xs, docs);
test.done();
})
})
};
You save save the above in tests/batch_deadlock.js
, install mongo drives with npm i mongodb
and finally run it with npx nodeunit tests/batch_deadlock.js
. It should deadlock when running with node version >= 10, but passes when ran with node 8.x.
I’ve also left a few comments in the code which show some tweaks that make the deadlock go away.
Issue Analytics
- State:
- Created 4 years ago
- Reactions:1
- Comments:13 (5 by maintainers)
Top Results From Across the Web
Issues · caolan/highland - GitHub
High-level streams library for Node.js and the browser - Issues ... deadlock when using batch with flatMap and mongo stream as source (node...
Read more >Node.JS + mongo: .find().each() stopping after first batch
I have a standalone (command-line executed) node script, whose purpose is to iterate through all the documents in a large collection (several ...
Read more >Node.js v19.3.0 Documentation
Using strings as inputs to cryptographic APIs; Legacy streams API (prior to Node.js 0.10); Support for weak or compromised algorithms; CCM mode.
Read more >Change Streams & Triggers with Node.js Tutorial - MongoDB
Discover how to react to changes in your MongoDB database using change streams implemented in Node.js and Atlas triggers.
Read more >Spring Integration Reference Guide
Figure 5. An inbound channel adapter endpoint connects a source system to a MessageChannel . Message sources can be pollable (for ...
Read more >
Top Related Medium Post
No results found
Top Related StackOverflow Question
No results found
Troubleshoot Live Code
Lightrun enables developers to add logs, metrics and snapshots to live code - no restarts or redeploys required.
Start Free
Top Related Reddit Thread
No results found
Top Related Hackernoon Post
No results found
Top Related Tweet
No results found
Top Related Dev.to Post
No results found
Top Related Hashnode Post
No results found
Tried
_(mockMongoStream(docs))
with a custom subclass ofStream.Readable
and it worked without issue https://travis-ci.com/eccentric-j/highland-batch-repro/builds/151668807. I also tried_(_(docs).toNodeReadable({ objectMode: true }))
and it also worked without issue https://travis-ci.com/eccentric-j/highland-batch-repro/builds/151663566.This has me leaning towards the issue residing in mongodb’s
cursor.stream()
method. The next step is to use the mongo db stream and try consuming it in a batch Node Transform stream to see if I can reproduce the error without using Highland.Thanks for that info! I’m going to dig in today and see if I can pinpoint if it’s a highland issue specifically or something with the batched mongo stream.
As for
take
, I wasn’t sure what the intention was looking at the example. Based on the use of.toCallback
and the fact that this example batches it into an array then flattens it out again, I was wondering if the goal was to get a finite stream out of an infinite stream.