Pause does not work

Why is this important? MongoDB is in many cases the standard to node when it comes to Databases. However if events are emitted faster than Mongoose can handle them, It can result in duplication and or index errors and possibly more

Attempts at solutions and work arounds

1. Pausing the Stream Directly inside of the “record” event

stream = fs.createReadStream(filepath);
csvstream = csv.fromStream(stream).on("record", function(data){
stream.pause();
console.log("shouldn't see this more than once");
})

stream.pause() doesn’t seem to do anything 2) Going into old mode, then pausing directly

stream = fs.createReadStream(filepath);
stream.pause();
csvstream = csv.fromStream(stream).on("record", function(data){
stream.pause();
console.log("shouldn't see this more than once");
})
stream.resume();

stream.pause() doesn’t seem to do anything 3) using pause method of the Parser_Stream https://github.com/C2FO/fast-csv/blob/master/lib/parser_stream.js#L156

stream = fs.createReadStream(filepath);
csvstream = csv.fromStream(stream).on("record", function(data){
csvstream.pause();
console.log("shouldn't see this more than once");
})

csvstream.pause() doesn’t seem to do anything 4) combining both https://github.com/C2FO/fast-csv/blob/master/lib/parser_stream.js#L156

stream = fs.createReadStream(filepath);
stream.pause();
csvstream = csv.fromStream(stream).on("record", function(data){
stream.pause();
csvstream.pause();
console.log("shouldn't see this more than once");
})
stream.resume();

callback doesn’t seem to do anything 5) Using csvstream to resume

stream = fs.createReadStream(filepath);
stream.pause();
csvstream = csv.fromStream(stream).on("record", function(data){
stream.pause();
csvstream.pause();
console.log("shouldn't see this more than once");
})
csvstream.resume();

Process Doesn’t start https://github.com/C2FO/fast-csv/blob/master/lib/parser_stream.js#L163 6) Attempt to see if csvstream.pause() even matters (which it doesn’t)

stream = fs.createReadStream(filepath);
stream.pause();
csvstream = csv.fromStream(stream).on("record", function(data){
stream.pause();
console.log("shouldn't see this");
})
csvstream.pause();
stream.resume();

Doesn’t work 7) Using from path

csvstream = csv.fromPath(filepath).on("record", function(data){
csvstream.pause();
console.log("shouldn't see this more than once");
})

Doesn’t work 8) Checking if from path at least prevents it

csvstream = csv.fromPath(filepath).on("record", function(data){
console.log("shouldn't see this");
})
csvstream.pause();

Doesn’t work

[Xeoncross]

Can you point me in the right direction? I’m having some bugs with pausing in 2.4.1 where end is called while my callbacks are still working on processing from the data callbacks.

[cmmash]

I’m having the same issue as @Xeoncross .

My end callback gets fired before my last row has stopped processing.

Here’s how my code is laid out:

  /** Starts the CSV reading and record update row by row process */
  _processCSV() {
    this.csvReadStream = fastCsv.parseStream(this.s3CSVReadStream,
      {
        headers: true,
        strictColumnHandling: true,
        delimiter: this.taskData.csvDelimiter || ',',
        encoding: 'utf8',
      }
    );

    this.csvReadStream
      // where the actual csv row reading and import/update happens
      .on('data', this._processCSVRow())
      // to process errors while reading off the stream
      .on('error', this._processCSVError())
      // in case csv is malformed or our own validations fail (must be defined in <reader>.validate)
      .on('data-invalid', this._processCSVDataInvalid())
      // when CSV is fully read.
      .on('end', this._processCSVEnd());
  }


  /** Handles CSV parsed rows */
  _processCSVRow() {
    return async rowData => {
      console.log('\n##############################################################################');
      console.log('##############################################################################\n');
      this.csvRowCtr += 1;
      this.currentCSVRow = rowData;
      this.csvReadStream.pause();

      if (!this.detectedCSVHeaders) this.detectedCSVHeaders = Object.keys(rowData);

      if (this.endCSVReadingPostCancellation) {
        console.log('----- row skipped: ' + this.csvRowCtr + '; csv reading ended');
        this.csvReadStream.resume(); // todo - should be pause ? or remove altogether if not working
        return;
      }

      console.log(`=====> NEW CSV ROW # = ${  this.csvRowCtr}`);
      // console.log('\nRAW DATA: ', rowData, '\n');

      // sets endCSVReadingPostCancellation to true if task is cancelled
      await this._makeCancellationCheck();

      if (!this.cancelled) {
        const dbWriteSuccess = await this._performAPICall(rowData);
        this._updateCounters(dbWriteSuccess);
      }

      await this._reportProgress(this.csvRowCtr - 1);
      console.log('=====> REPORTED PROGRESS FOR ROW = ' + this.csvRowCtr)
      this.csvReadStream.resume();
    };
  }


  /** Processes the CSV reading end - runs post execution hooks, logs stats about success/failure rows. */
  _processCSVEnd() {
    return async totalRowsProcessed => {
      console.log('==> _processCSVEnd; totalRowsProcessed: ', totalRowsProcessed);
      if (totalRowsProcessed === this.successRowsCtr && totalRowsProcessed === this.successRowsCtr) {
        this.endStatus = STATUS_SUCCEEDED;
      } else if (this.successRowsCtr && this.successRowsCtr > 0) {
        this.endStatus = STATUS_SUCCEEDED_PARTIALLY;
      } else {
        this.endStatus = STATUS_FAILED;
      }

      console.log('\n------> _processCSVEnd CSV PROCESSING ENDED:\n  csvRowCtr: ' + this.csvRowCtr +
        '  \n\n=======> _processCSVEnd validationErrors: ', JSON.stringify(this.validationErrors), '\n');

      if (!this.cancelled && this.errorCSVUploader && this.errorCSVWriter && this.validationErrorRowsCtr) {
        // finish the error csv writer, and it will automatically finish the s3 upload, in whose uploadCallback
        // we will then run the postExecHooks.
        console.log('======> _processCSVEnd STOPPING ERROR CSV WRITER')
        // postExecHooks will be called automatically in s3 upload finish callback
        this._stopErrorCSVWriter();
      } else {
        console.log('=========> _processCSVEnd calling postExecHooks')
        await this._runPostExecutionHooks();
      }
    };
  }