Getting results back from batch query job

I have something like this, where I want to send a query as a bulk batch job.

  const job = conn.bulk.createJob('Account', 'query');
  var batch = job.createBatch();
  let endBatch;
  batch.execute(theQuery);
  batch.on('queue', batchInfo => {
    console.log('in queue');
    endBatch = conn.bulk.job(batchInfo.jobId).batch(batchInfo.batchId);
    endBatch.poll(500, 20000);

    endBatch.on('response', response => {
      console.log('response', response);
      endBatch.check().then(yo => {
        console.log('check', yo);
      });
      endBatch
        .retrieve()
        .then(results => {
          console.log('end', results);
        })
        .catch(handleError);
    });
  });

However, all I ever get back from the retrieve() is something like this, without the actual results themselves.

end [ { id: '75261000008FPXW',
 batchId: '7516100000GnBSwAAN',
 jobId: '7506100000CtuBwAAJ' } ]

My check also shows that everything has completed, but I never get the results back:

check { '$': { xmlns: 'http://www.force.com/2009/06/asyncapi/dataload' },
 id: '7516100000GnBSwAAN',
 jobId: '7506100000CtuBwAAJ',
 state: 'Completed',
 createdDate: '2018-08-01T18:36:12.000Z',
 systemModstamp: '2018-08-01T18:36:12.000Z',
 numberRecordsProcessed: '2',
 numberRecordsFailed: '0',
 totalProcessingTime: '0',
 apiActiveProcessingTime: '0',
 apexProcessingTime: '0' }

Looking at the actual code in api_bulk.js, it seems like results are never actually getting returned. - Am I missing something here? Is there another event I should be listening for?

[mprown]

Does anyone have a recommendation on how to retrieve the results when working in typescript?

[aguynamedben]

@supermarioim Here’s how we’re doing large fetches of Salesforce tables via Bulk API, if this helps. We decided to just handle the RecordStream ourselves to have finer control of what’s happening, see the comments.

We’re still debugging some occasional Salesforce polling timeouts that seem customer-specific (i.e. maybe certain Salesforce instances have a ton of records, the Bulk API queue is backed up, etc.). We might resort to getting fewer records via the REST API in that case.

Hopefully this helps!

export default class SalesforceBulkAPI {
  static async getSalesforceRecords(client, table) {
    const countSoql = `SELECT COUNT() FROM ${table.tableName}`;
    const result = await client.query(countSoql);
    const { totalSize } = result;

    const limit = 10000;
    const soql = tableExportSoql(table, limit);
    log.info(`There are ${totalSize} ${table.tableName} records (limit is ${limit})`);

    /**
     * Don't trust jsforce's stream handling, use our own
     * https://github.com/jsforce/jsforce/issues/691#issuecomment-341107251
     * https://stackoverflow.com/questions/10623798/writing-node-js-stream-into-a-string-variable/35530615#35530615
     */

    const t1 = performance.now();

    const csvParser = csvParse({ delimiter: ',', columns: true });
    // We still need recordStream to listen for errors. We'll access the stream
    // directly though, bypassing jsforce's RecordStream.Parsable
    const recordStream = client.bulk.query(soql);
    const readStream = recordStream.stream();
    readStream.pipe(csvParser);

    const records = [];
    let recordBatches = 0;

    // https://csv.js.org/parse/api/
    csvParser.on('readable', () => {
      let record;
      while (record = csvParser.read()) { // eslint-disable-line no-cond-assign
        records.push(record);
      }
      recordBatches += 1;
    });

    return new Promise((resolve, reject) => {
      recordStream.on('error', (error) => {
        log.error(error);
        if (error.name === 'PollingTimeout') {
          // https://github.com/jsforce/jsforce/blob/a091f2f72b0fa5bb784cf098b60988f707e720ef/lib/api/bulk.js#L574
          reject(new Error(`The Salesforce Bulk API had a polling timeout`));
        } else {
          reject(new Error(`Couldn't download results from Salesforce Bulk API`));
        }
      });

      csvParser.on('error', (error) => {
        log.error(error);
        reject(new Error(`Couldn't parse results from Salesforce Bulk API`));
      });

      csvParser.on('end', async () => {
        const t2 = performance.now();
        log.info(`Received ${records.length} of ${totalSize} (limit ${limit}) ${table.tableName} records in ${recordBatches} batches in ${timeHelpers.msString(t2, t1)}`);
        resolve(records);
      });

      // Throw fake PollingTimeout error from jsforce
      if (false && isDev) {
        const err = new Error(`Polling time out. Job Id = fake , batch Id = fake`);
        err.name = 'PollingTimeout';
        err.jobId = 'abc';
        err.batchId = 'def';
        recordStream.emit('error', err);
      }
    });
  }
}

Usage would be something like:

const recordsForTable = SalesforceBulkAPI.getSalesforceRecords(jsforceClient, 'Account');

Note: During these requests, an error with error.name === 'invalid_grant' maybe be triggered, so watch out for that! We wrap our calls to getSalesforceRecords with some error handling that catches invalid_grant and triggers our apps token refresh logic.