help with highland configurable consumer

I have thee following code with a file test.info containing some numbers each on their own line. I expect this to output a series of lines Doubled = then the doubled number. Instead I’m getting the error that follows. It is a little funky, I admit, but I am working with some legacy code and want to be able to conditionally swap in a highland based consumer, so can’t switch it all out to highland. Any ideas? (Is there a better spot for questions like these? Slack? IRC?)

const _ = require('highland');
const fs = require('fs');
const Pump = require('util').promisify(require('stream').pipeline);

const double = x => x*2;

const consumer = _.curry(({ func, msg }, data) => _(data)
    .map(func)
    .each(x => console.log(msg, x))
    .done(()=> console.log('done consuming...'))
);

const main = async () => {
    const fileStream = fs.createReadStream('test.info');
    const publisher = consumer({func: double, msg: 'Doubled = '});

    await Pump(fileStream, publisher);

    console.log('DONE');
};

main();

(node:42596) UnhandledPromiseRejectionWarning: TypeError: Cannot read property 'on' of undefined
    at destroyer (internal/streams/pipeline.js:23:10)
    at internal/streams/pipeline.js:78:12
    at Array.map (<anonymous>)
    at pipeline (internal/streams/pipeline.js:75:28)
    at internal/util.js:278:30
    at new Promise (<anonymous>)
    at pipeline (internal/util.js:277:12)
    at main (/Users/jdietrich/work/ipsis-standardcsv/src/test.js:18:11)
    at Object.<anonymous> (/Users/jdietrich/work/ipsis-standardcsv/src/test.js:23:1)
    at Module._compile (internal/modules/cjs/loader.js:959:30)

[eccentric-j]

The CI helped prove that solution will not work on versions before node 14.

https://github.com/eccentric-j/highland-pipline-repro/actions/runs/144114428

The most straight forward solution that works with all 3 node versions:

const _ = require('highland');
const fs = require('fs');
const Pump = require('util').promisify(require('stream').pipeline);

const double = x => x*2;

async function main () {
    await fs.createReadStream('test.info', { encoding: 'utf8' })
        .pipe(_())
        .split()
        .map(double)
        .tap(x => console.log('Doubled = ', x))
        .collect()
        .toPromise(Promise)

    console.log('DONE');
}

main()
    .catch(err => {
        console.error(err);
        process.exit(1);
    });

https://github.com/eccentric-j/highland-pipline-repro/actions/runs/144122089

Finally, there’s a solution that works about the way you intended it:

const _ = require('highland');
const fs = require('fs');
const stream = require("stream");
const Pump = require('util').promisify(stream.pipeline);

const double = x => x*2;

const consumer = _.curry(({ func, msg }, data) => _(data)
    .split()
    .map(func)
    .tap(x => console.log(msg, x))
);

function transformStream (streamFn, opts) {
    const input = _();
    const output = streamFn(input);
    let subscribed = false;

    const xfStream = new stream.Transform(Object.assign({
        objectMode: true,
        transform (chunk, encoding, callback) {
            // we only want to create this subscription once
            if (!subscribed) {
                output
                  .errors((err, push) => {
                      // end the highland stream
                      push(_.nil);
                      // destroy outer transform stream with error
                      this.destroy(err);
                  })
                  .each(x => this.push(x))
                subscribed = true;
            }
            input.write(chunk);
            callback();
        },

        flush (callback) {
            input.end();
        }
    }, opts))

    return xfStream;
}

const main = async () => {
    const fileStream = fs.createReadStream('test.info', { encoding:  'utf8' });
    const publisher = consumer({ func: double, msg: 'Doubled = ' });

    await Pump(fileStream, transformStream(publisher));

    console.log('DONE');
};


main()
    .catch(err => {
        console.error(err);
        process.exit(1);
    });

Which does work on the last 3 node versions as well https://github.com/eccentric-j/highland-pipline-repro/actions/runs/144206922

[jcdietrich]

Thank you!