Some errors are retried until max but then returned as successful

I see a weird behaviour when using the retry mechanism.

I’m doing request to a server that sometimes response with 500, internal server error, and when it does, I usually get those 500 for all retried requests.

The problem I see is that when the retries are done, and even the last request failed, I get called in the then function and not catch. I have tried the callback based version as well, and even here the err is empty.

Any other error, like pulling the network cable is handled fine.

The config is as follows:

  request({
    method: 'POST',
    uri: url,
    body: params,
    auth: { username: username, password: password },
    json: true,
    timeout: 5000,
    maxAttempts: 5, // try 5 times
    retryDelay: 1000, // wait for 1s before trying again
    retryStrategy: request.RetryStrategies.HTTPOrNetworkError // (default) retry on 5xx or network errors
  }

And this is the response I get to the then function after last failed retry with sensitive data replaced with ***:

IncomingMessage {
  _readableState: 
   ReadableState {
     objectMode: false,
     highWaterMark: 16384,
     buffer: [],
     length: 0,
     pipes: null,
     pipesCount: 0,
     flowing: true,
     ended: true,
     endEmitted: true,
     reading: false,
     sync: false,
     needReadable: false,
     emittedReadable: false,
     readableListening: false,
     defaultEncoding: 'utf8',
     ranOut: false,
     awaitDrain: 0,
     readingMore: false,
     decoder: null,
     encoding: null,
     resumeScheduled: false },
  readable: false,
  domain: null,
  _events: 
   { end: [ [Function: responseOnEnd], [Function], [Function], [Function] ],
     close: [ [Function], [Function] ],
     data: [Function],
     error: [Function] },
  _eventsCount: 4,
  _maxListeners: undefined,
  socket: 
   TLSSocket {
     _tlsOptions: 
      { pipe: null,
        secureContext: [Object],
        isServer: false,
        requestCert: true,
        rejectUnauthorized: true,
        session: ***,
        NPNProtocols: undefined,
        ALPNProtocols: undefined,
        requestOCSP: undefined },
     _secureEstablished: true,
     _securePending: false,
     _newSessionPending: false,
     _controlReleased: true,
     _SNICallback: null,
     servername: null,
     npnProtocol: undefined,
     alpnProtocol: false,
     authorized: true,
     authorizationError: null,
     encrypted: true,
     _events: 
      { close: [Object],
        end: [Object],
        finish: [Function: onSocketFinish],
        _socketEnd: [Function: onSocketEnd],
        secure: [Function],
        free: [Function: onFree],
        agentRemove: [Function: onRemove],
        drain: [Function: ondrain],
        error: [Object],
        data: [Function: socketOnData],
        timeout: [Object] },
     _eventsCount: 11,
     _connecting: false,
     _hadError: false,
     _handle: null,
     _parent: null,
     _host: '***',
     _readableState: 
      ReadableState {
        objectMode: false,
        highWaterMark: 16384,
        buffer: [],
        length: 0,
        pipes: null,
        pipesCount: 0,
        flowing: true,
        ended: true,
        endEmitted: true,
        reading: false,
        sync: false,
        needReadable: false,
        emittedReadable: false,
        readableListening: false,
        defaultEncoding: 'utf8',
        ranOut: false,
        awaitDrain: 0,
        readingMore: false,
        decoder: null,
        encoding: null,
        resumeScheduled: false },
     readable: false,
     domain: null,
     _maxListeners: 0,
     _writableState: 
      WritableState {
        objectMode: false,
        highWaterMark: 16384,
        needDrain: false,
        ending: true,
        ended: true,
        finished: true,
        decodeStrings: false,
        defaultEncoding: 'utf8',
        length: 0,
        writing: false,
        corked: 0,
        sync: false,
        bufferProcessing: false,
        onwrite: [Function],
        writecb: null,
        writelen: 0,
        bufferedRequest: null,
        lastBufferedRequest: null,
        pendingcb: 0,
        prefinished: true,
        errorEmitted: false },
     writable: false,
     allowHalfOpen: false,
     destroyed: true,
     bytesRead: 346,
     _bytesDispatched: 375,
     _sockname: null,
     _pendingData: null,
     _pendingEncoding: '',
     ssl: 
      TLSWrap {
        _externalStream: {},
        fd: -22,
        _parent: [Object],
        _parentWrap: undefined,
        _secureContext: [Object],
        reading: true,
        owner: [Circular],
        onread: [Function: noop],
        writeQueueSize: 1,
        onhandshakestart: [Function],
        onhandshakedone: [Function: bound ],
        onocspresponse: [Function: bound onocspresponse],
        onerror: [Function] },
     server: undefined,
     _requestCert: true,
     _rejectUnauthorized: true,
     parser: null,
     _httpMessage: 
      ClientRequest {
        domain: null,
        _events: [Object],
        _eventsCount: 5,
        _maxListeners: undefined,
        output: [],
        outputEncodings: [],
        outputCallbacks: [],
        outputSize: 0,
        writable: true,
        _last: true,
        chunkedEncoding: false,
        shouldKeepAlive: false,
        useChunkedEncodingByDefault: true,
        sendDate: false,
        _removedHeader: [Object],
        _contentLength: null,
        _hasBody: true,
        _trailer: '',
        finished: true,
        _headerSent: true,
        socket: [Circular],
        connection: [Circular],
        _header: '***',
        _headers: [Object],
        _headerNames: [Object],
        _onPendingData: null,
        agent: [Object],
        socketPath: undefined,
        method: 'POST',
        path: '***',
        timeoutCb: [Function: emitTimeout],
        parser: null,
        res: [Circular] },
     _idleTimeout: -1,
     _idleNext: null,
     _idlePrev: null,
     _idleStart: 28882,
     read: [Function],
     _consuming: true,
     write: [Function: writeAfterFIN] },
  connection: 
   TLSSocket {
     _tlsOptions: 
      { pipe: null,
        secureContext: [Object],
        isServer: false,
        requestCert: true,
        rejectUnauthorized: true,
        session: ***,
        NPNProtocols: undefined,
        ALPNProtocols: undefined,
        requestOCSP: undefined },
     _secureEstablished: true,
     _securePending: false,
     _newSessionPending: false,
     _controlReleased: true,
     _SNICallback: null,
     servername: null,
     npnProtocol: undefined,
     alpnProtocol: false,
     authorized: true,
     authorizationError: null,
     encrypted: true,
     _events: 
      { close: [Object],
        end: [Object],
        finish: [Function: onSocketFinish],
        _socketEnd: [Function: onSocketEnd],
        secure: [Function],
        free: [Function: onFree],
        agentRemove: [Function: onRemove],
        drain: [Function: ondrain],
        error: [Object],
        data: [Function: socketOnData],
        timeout: [Object] },
     _eventsCount: 11,
     _connecting: false,
     _hadError: false,
     _handle: null,
     _parent: null,
     _host: '***',
     _readableState: 
      ReadableState {
        objectMode: false,
        highWaterMark: 16384,
        buffer: [],
        length: 0,
        pipes: null,
        pipesCount: 0,
        flowing: true,
        ended: true,
        endEmitted: true,
        reading: false,
        sync: false,
        needReadable: false,
        emittedReadable: false,
        readableListening: false,
        defaultEncoding: 'utf8',
        ranOut: false,
        awaitDrain: 0,
        readingMore: false,
        decoder: null,
        encoding: null,
        resumeScheduled: false },
     readable: false,
     domain: null,
     _maxListeners: 0,
     _writableState: 
      WritableState {
        objectMode: false,
        highWaterMark: 16384,
        needDrain: false,
        ending: true,
        ended: true,
        finished: true,
        decodeStrings: false,
        defaultEncoding: 'utf8',
        length: 0,
        writing: false,
        corked: 0,
        sync: false,
        bufferProcessing: false,
        onwrite: [Function],
        writecb: null,
        writelen: 0,
        bufferedRequest: null,
        lastBufferedRequest: null,
        pendingcb: 0,
        prefinished: true,
        errorEmitted: false },
     writable: false,
     allowHalfOpen: false,
     destroyed: true,
     bytesRead: 346,
     _bytesDispatched: 375,
     _sockname: null,
     _pendingData: null,
     _pendingEncoding: '',
     ssl: 
      TLSWrap {
        _externalStream: {},
        fd: -22,
        _parent: [Object],
        _parentWrap: undefined,
        _secureContext: [Object],
        reading: true,
        owner: [Circular],
        onread: [Function: noop],
        writeQueueSize: 1,
        onhandshakestart: [Function],
        onhandshakedone: [Function: bound ],
        onocspresponse: [Function: bound onocspresponse],
        onerror: [Function] },
     server: undefined,
     _requestCert: true,
     _rejectUnauthorized: true,
     parser: null,
     _httpMessage: 
      ClientRequest {
        domain: null,
        _events: [Object],
        _eventsCount: 5,
        _maxListeners: undefined,
        output: [],
        outputEncodings: [],
        outputCallbacks: [],
        outputSize: 0,
        writable: true,
        _last: true,
        chunkedEncoding: false,
        shouldKeepAlive: false,
        useChunkedEncodingByDefault: true,
        sendDate: false,
        _removedHeader: [Object],
        _contentLength: null,
        _hasBody: true,
        _trailer: '',
        finished: true,
        _headerSent: true,
        socket: [Circular],
        connection: [Circular],
        _header: '***',
        _headers: [Object],
        _headerNames: [Object],
        _onPendingData: null,
        agent: [Object],
        socketPath: undefined,
        method: 'POST',
        path: '***',
        timeoutCb: [Function: emitTimeout],
        parser: null,
        res: [Circular] },
     _idleTimeout: -1,
     _idleNext: null,
     _idlePrev: null,
     _idleStart: 28882,
     read: [Function],
     _consuming: true,
     write: [Function: writeAfterFIN] },
  httpVersionMajor: 1,
  httpVersionMinor: 1,
  httpVersion: '1.1',
  complete: true,
  headers: 
   { 'content-type': 'text/xml',
     'x-backside-transport': 'FAIL FAIL',
     connection: 'close' },
  rawHeaders: 
   [ 'Content-Type',
     'text/xml',
     'X-Backside-Transport',
     'FAIL FAIL',
     'Connection',
     'close' ],
  trailers: {},
  rawTrailers: [],
  upgrade: false,
  url: '',
  method: null,
  statusCode: 500,
  statusMessage: 'Internal Server Error',
  client: 
   TLSSocket {
     _tlsOptions: 
      { pipe: null,
        secureContext: [Object],
        isServer: false,
        requestCert: true,
        rejectUnauthorized: true,
        session: ***,
        NPNProtocols: undefined,
        ALPNProtocols: undefined,
        requestOCSP: undefined },
     _secureEstablished: true,
     _securePending: false,
     _newSessionPending: false,
     _controlReleased: true,
     _SNICallback: null,
     servername: null,
     npnProtocol: undefined,
     alpnProtocol: false,
     authorized: true,
     authorizationError: null,
     encrypted: true,
     _events: 
      { close: [Object],
        end: [Object],
        finish: [Function: onSocketFinish],
        _socketEnd: [Function: onSocketEnd],
        secure: [Function],
        free: [Function: onFree],
        agentRemove: [Function: onRemove],
        drain: [Function: ondrain],
        error: [Object],
        data: [Function: socketOnData],
        timeout: [Object] },
     _eventsCount: 11,
     _connecting: false,
     _hadError: false,
     _handle: null,
     _parent: null,
     _host: '***',
     _readableState: 
      ReadableState {
        objectMode: false,
        highWaterMark: 16384,
        buffer: [],
        length: 0,
        pipes: null,
        pipesCount: 0,
        flowing: true,
        ended: true,
        endEmitted: true,
        reading: false,
        sync: false,
        needReadable: false,
        emittedReadable: false,
        readableListening: false,
        defaultEncoding: 'utf8',
        ranOut: false,
        awaitDrain: 0,
        readingMore: false,
        decoder: null,
        encoding: null,
        resumeScheduled: false },
     readable: false,
     domain: null,
     _maxListeners: 0,
     _writableState: 
      WritableState {
        objectMode: false,
        highWaterMark: 16384,
        needDrain: false,
        ending: true,
        ended: true,
        finished: true,
        decodeStrings: false,
        defaultEncoding: 'utf8',
        length: 0,
        writing: false,
        corked: 0,
        sync: false,
        bufferProcessing: false,
        onwrite: [Function],
        writecb: null,
        writelen: 0,
        bufferedRequest: null,
        lastBufferedRequest: null,
        pendingcb: 0,
        prefinished: true,
        errorEmitted: false },
     writable: false,
     allowHalfOpen: false,
     destroyed: true,
     bytesRead: 346,
     _bytesDispatched: 375,
     _sockname: null,
     _pendingData: null,
     _pendingEncoding: '',
     ssl: 
      TLSWrap {
        _externalStream: {},
        fd: -22,
        _parent: [Object],
        _parentWrap: undefined,
        _secureContext: [Object],
        reading: true,
        owner: [Circular],
        onread: [Function: noop],
        writeQueueSize: 1,
        onhandshakestart: [Function],
        onhandshakedone: [Function: bound ],
        onocspresponse: [Function: bound onocspresponse],
        onerror: [Function] },
     server: undefined,
     _requestCert: true,
     _rejectUnauthorized: true,
     parser: null,
     _httpMessage: 
      ClientRequest {
        domain: null,
        _events: [Object],
        _eventsCount: 5,
        _maxListeners: undefined,
        output: [],
        outputEncodings: [],
        outputCallbacks: [],
        outputSize: 0,
        writable: true,
        _last: true,
        chunkedEncoding: false,
        shouldKeepAlive: false,
        useChunkedEncodingByDefault: true,
        sendDate: false,
        _removedHeader: [Object],
        _contentLength: null,
        _hasBody: true,
        _trailer: '',
        finished: true,
        _headerSent: true,
        socket: [Circular],
        connection: [Circular],
        _header: '***',
        _headers: [Object],
        _headerNames: [Object],
        _onPendingData: null,
        agent: [Object],
        socketPath: undefined,
        method: 'POST',
        path: '***',
        timeoutCb: [Function: emitTimeout],
        parser: null,
        res: [Circular] },
     _idleTimeout: -1,
     _idleNext: null,
     _idlePrev: null,
     _idleStart: 28882,
     read: [Function],
     _consuming: true,
     write: [Function: writeAfterFIN] },
  _consuming: true,
  _dumped: false,
  req: 
   ClientRequest {
     domain: null,
     _events: 
      { socket: [Object],
        timeout: [Object],
        response: [Function: bound ],
        error: [Function: bound ],
        drain: [Function] },
     _eventsCount: 5,
     _maxListeners: undefined,
     output: [],
     outputEncodings: [],
     outputCallbacks: [],
     outputSize: 0,
     writable: true,
     _last: true,
     chunkedEncoding: false,
     shouldKeepAlive: false,
     useChunkedEncodingByDefault: true,
     sendDate: false,
     _removedHeader: { 'content-length': false },
     _contentLength: null,
     _hasBody: true,
     _trailer: '',
     finished: true,
     _headerSent: true,
     socket: 
      TLSSocket {
        _tlsOptions: [Object],
        _secureEstablished: true,
        _securePending: false,
        _newSessionPending: false,
        _controlReleased: true,
        _SNICallback: null,
        servername: null,
        npnProtocol: undefined,
        alpnProtocol: false,
        authorized: true,
        authorizationError: null,
        encrypted: true,
        _events: [Object],
        _eventsCount: 11,
        _connecting: false,
        _hadError: false,
        _handle: null,
        _parent: null,
        _host: '***',
        _readableState: [Object],
        readable: false,
        domain: null,
        _maxListeners: 0,
        _writableState: [Object],
        writable: false,
        allowHalfOpen: false,
        destroyed: true,
        bytesRead: 346,
        _bytesDispatched: 375,
        _sockname: null,
        _pendingData: null,
        _pendingEncoding: '',
        ssl: [Object],
        server: undefined,
        _requestCert: true,
        _rejectUnauthorized: true,
        parser: null,
        _httpMessage: [Circular],
        _idleTimeout: -1,
        _idleNext: null,
        _idlePrev: null,
        _idleStart: 28882,
        read: [Function],
        _consuming: true,
        write: [Function: writeAfterFIN] },
     connection: 
      TLSSocket {
        _tlsOptions: [Object],
        _secureEstablished: true,
        _securePending: false,
        _newSessionPending: false,
        _controlReleased: true,
        _SNICallback: null,
        servername: null,
        npnProtocol: undefined,
        alpnProtocol: false,
        authorized: true,
        authorizationError: null,
        encrypted: true,
        _events: [Object],
        _eventsCount: 11,
        _connecting: false,
        _hadError: false,
        _handle: null,
        _parent: null,
        _host: '***',
        _readableState: [Object],
        readable: false,
        domain: null,
        _maxListeners: 0,
        _writableState: [Object],
        writable: false,
        allowHalfOpen: false,
        destroyed: true,
        bytesRead: 346,
        _bytesDispatched: 375,
        _sockname: null,
        _pendingData: null,
        _pendingEncoding: '',
        ssl: [Object],
        server: undefined,
        _requestCert: true,
        _rejectUnauthorized: true,
        parser: null,
        _httpMessage: [Circular],
        _idleTimeout: -1,
        _idleNext: null,
        _idlePrev: null,
        _idleStart: 28882,
        read: [Function],
        _consuming: true,
        write: [Function: writeAfterFIN] },
     _header: '***',
     _headers: 
      { host: '***',
        authorization: '***',
        accept: 'application/json',
        'content-type': 'application/json',
        'content-length': 120 },
     _headerNames: 
      { host: 'host',
        authorization: 'authorization',
        accept: 'accept',
        'content-type': 'content-type',
        'content-length': 'content-length' },
     _onPendingData: null,
     agent: 
      Agent {
        domain: null,
        _events: [Object],
        _eventsCount: 1,
        _maxListeners: undefined,
        defaultPort: 443,
        protocol: 'https:',
        options: [Object],
        requests: {},
        sockets: [Object],
        freeSockets: {},
        keepAliveMsecs: 1000,
        keepAlive: false,
        maxSockets: Infinity,
        maxFreeSockets: 256,
        maxCachedSessions: 100,
        _sessionCache: [Object] },
     socketPath: undefined,
     method: 'POST',
     path: '***',
     timeoutCb: [Function: emitTimeout],
     parser: null,
     res: [Circular] },
  request: 
   Request {
     domain: null,
     _events: 
      { error: [Function: bound ],
        complete: [Function: bound ],
        pipe: [Function],
        end: [Object],
        data: [Function] },
     _eventsCount: 5,
     _maxListeners: undefined,
     method: 'POST',
     uri: 
      Url {
        protocol: 'https:',
        slashes: true,
        auth: null,
        host: '***',
        port: 443,
        hostname: '***',
        hash: null,
        search: null,
        query: null,
        pathname: '***',
        path: '***',
        href: '***' },
     body: '***',
     timeout: 10000,
     maxAttempts: 5,
     retryDelay: 1000,
     retryStrategy: [Function: HTTPError],
     fullResponse: true,
     promiseFactory: [Function: defaultPromiseFactory],
     callback: [Function],
     readable: true,
     writable: true,
     explicitMethod: true,
     _qs: 
      Querystring {
        request: [Circular],
        lib: [Object],
        useQuerystring: undefined,
        parseOptions: {},
        stringifyOptions: {} },
     _auth: 
      Auth {
        request: [Circular],
        hasAuth: true,
        sentAuth: true,
        bearerToken: null,
        user: '***',
        pass: '***' },
     _oauth: OAuth { request: [Circular], params: null },
     _multipart: 
      Multipart {
        request: [Circular],
        boundary: '6b8fcb2b-95fb-4465-9682-a05ea8b32c25',
        chunked: false,
        body: null },
     _redirect: 
      Redirect {
        request: [Circular],
        followRedirect: true,
        followRedirects: true,
        followAllRedirects: false,
        allowRedirect: [Function],
        maxRedirects: 10,
        redirects: [],
        redirectsFollowed: 0,
        removeRefererHeader: false },
     _tunnel: 
      Tunnel {
        request: [Circular],
        proxyHeaderWhiteList: [Object],
        proxyHeaderExclusiveList: [] },
     headers: 
      { authorization: '***',
        accept: 'application/json',
        'content-type': 'application/json',
        'content-length': 120 },
     setHeader: [Function],
     hasHeader: [Function],
     getHeader: [Function],
     removeHeader: [Function],
     localAddress: undefined,
     pool: {},
     dests: [],
     __isRequestRequest: true,
     _callback: [Function: bound ],
     proxy: null,
     tunnel: true,
     setHost: true,
     originalCookieHeader: undefined,
     _disableCookies: true,
     _jar: undefined,
     port: 443,
     host: '***',
     path: '***',
     _json: true,
     httpModule: 
      { Server: [Object],
        createServer: [Function],
        globalAgent: [Object],
        Agent: [Object],
        request: [Function],
        get: [Function] },
     agentClass: { [Function: Agent] super_: [Object] },
     agent: 
      Agent {
        domain: null,
        _events: [Object],
        _eventsCount: 1,
        _maxListeners: undefined,
        defaultPort: 443,
        protocol: 'https:',
        options: [Object],
        requests: {},
        sockets: [Object],
        freeSockets: {},
        keepAliveMsecs: 1000,
        keepAlive: false,
        maxSockets: Infinity,
        maxFreeSockets: 256,
        maxCachedSessions: 100,
        _sessionCache: [Object] },
     _started: true,
     href: '***',
     req: 
      ClientRequest {
        domain: null,
        _events: [Object],
        _eventsCount: 5,
        _maxListeners: undefined,
        output: [],
        outputEncodings: [],
        outputCallbacks: [],
        outputSize: 0,
        writable: true,
        _last: true,
        chunkedEncoding: false,
        shouldKeepAlive: false,
        useChunkedEncodingByDefault: true,
        sendDate: false,
        _removedHeader: [Object],
        _contentLength: null,
        _hasBody: true,
        _trailer: '',
        finished: true,
        _headerSent: true,
        socket: [Object],
        connection: [Object],
        _header: '***',
        _headers: [Object],
        _headerNames: [Object],
        _onPendingData: null,
        agent: [Object],
        socketPath: undefined,
        method: 'POST',
        path: '***',
        timeoutCb: [Function: emitTimeout],
        parser: null,
        res: [Circular] },
     timeoutTimer: null,
     ntick: true,
     response: [Circular],
     originalHost: '***',
     originalHostHeaderName: 'host',
     responseContent: [Circular],
     _destdata: true,
     _ended: true,
     _callbackCalled: true },
  toJSON: [Function: responseToJSON],
  caseless: 
   Caseless {
     dict: 
      { 'content-type': 'text/xml',
        'x-backside-transport': 'FAIL FAIL',
        connection: 'close' } },
  read: [Function],
  body: '<?xml version=\'1.0\' ?>\n<env:Envelope xmlns:env=\'http://schemas.xmlsoap.org/soap/envelope/\'>\n<env:Body>\n<env:Fault>\n<faultcode>env:Client</faultcode>\n<faultstring>Internal Error</faultstring>\n</env:Fault>\n</env:Body>\n</env:Envelope>\n',
  attempts: 5 }

Any ideas?

[Jokero]

@adrukh My solution:

'use strict';

const request = require('requestretry');

module.exports = request.defaults({
    json: true,
    maxAttempts: 5,
    retryDelay: 2000,
    timeout: 3000,
    promiseFactory(resolver) {
        return new Promise(resolver)
            .then(({ statusCode, body }) => {
                if (statusCode >= 400) {
                    const error = new Error();
                    error.body = body;
                    error.statusCode = statusCode;
                    return Promise.reject(error);
                }

                return body;
            })
    }
});

[adrukh]

@FGRibreau thanks for the suggestion and clarification on the intended design of this helpful module 😃

@Jokero perhaps the following will be of use to you, this is what I ended up doing to overcome this. Not too elegant, but does the trick:

return requestretry({ method, uri, body, json: true }).then(resp => {
    if (resp.statusCode >= 400) {
      const error = new Error(resp.statusCode); // consider using `http-errors` here for verbosity
      error.body = resp.body;
      error.statusCode = resp.statusCode;
      return Promise.reject(error);
    }
    return resp;
  });