Reason for XHR request and response inside JavaScript workers no being logged.

Hi,

I am trying to log all the XHR requests and responses from a website. However, I see some XHR requests and responses not captured by Network.requestWillBeSent and Network.responseReceived event handlers. What could be the reason for this?

Note that, I can see this XHR requests and responses, that my script failed to capture, in the network tab in my chrome debug tools. So what could be the reason for the script to miss this request and response?

Following is the code snippet that I am using:

Network.requestWillBeSent((params) => {
        fs.appendFile(requests, params.requestId +
        ',' + params.request.url + '\n', function(err){});
    });
    Network.responseReceived((params) => {   
      	  downloadFile[params.requestId] = params.type;
    });

  Network.loadingFinished((params) => {
        if (downloadFile[params.requestId]) {
            Network.getResponseBody({
                requestId: params.requestId
            }).then(response => {
                fs.appendFileSync(path.join(folder, downloadFile[params.requestId] + "-" + params.requestId),
                response.body, function(err){});
            });
        }
    });

[cyrus-and]

This happens because those requests are performed inside JavaScript workers and from a Chrome perspective they are separate (sub) targets. Bad news is that handling this case requires to write a much more complex code since you cannot use the usual JavaScript API, but if you’re just interested in events from workers you can get away pretty easily:

const CDP = require('chrome-remote-interface');

CDP(async (client) => {
    const {Page, Target} = client;

    Target.attachedToTarget(({sessionId, targetInfo: {url}}) => {
        console.log(`ATTACHED ${sessionId}: ${url}`);
        // enable networks events in this target, you can't use the regular API
        // here, so you have to manually build the messages and take care of
        // ids; it's not really a big deal unless you need to match commands and
        // responses
        Target.sendMessageToTarget({
            sessionId,
            message: JSON.stringify({ // <-- a JSON string!
                id: 1,
                method: 'Network.enable'
            })
        });
    });

    Target.receivedMessageFromTarget(({sessionId, message}) => {
        const {id, method, params} = JSON.parse(message); // <-- a JSON string!
        // you can handle massages from the target here; the same apply about
        // the fact that you can't use the regular API here
        switch (method) {
        case 'Network.requestWillBeSent':
            {
                const {request: {url}} = params;
                console.log(`FROM ${sessionId}: ${url}`);
                break;
            }
        default:
            // skip...
        }
    });

    try {
        // automatically attach to newly created target so you don't have to do
        // that manually; you might not interested in all of them though...
        await Target.setAutoAttach({
            autoAttach: true,
            waitForDebuggerOnStart: false
        });
        await Page.navigate({url: 'https://websightjs.com/'});
    } catch (err) {
        console.error(err);
    }
}).on('error', (err) => {
    console.error(err);
});

The output will be something like:

ATTACHED dedicated:5678.1-1: https://websightjs.com/asm-worker.js
ATTACHED dedicated:5678.2-2: https://websightjs.com/wasm-worker.js
ATTACHED dedicated:5678.3-3: https://websightjs.com/js-worker.js
FROM dedicated:5678.2-2: https://websightjs.com/cv-wasm.data
FROM dedicated:5678.2-2: https://websightjs.com/cv-wasm.wasm
FROM dedicated:5678.1-1: https://websightjs.com/cv.data

[goelayu]

This issue is closed, yet I’m posting here in case someone else got stuck with this as I did. Apparently just connecting to the service worker doesn’t suffice, as it further uses a dedicated worker to actually make the network requests. So you’ll have to send a message from the main target to the service worker, which in turn should another message to the dedicated worker. Just to extend @cyrus-and 's solution, something like this should work

           Target.attachedToTarget(({sessionId, targetInfo: {url}}) => {
                console.log(`ATTACHED ${sessionId}: ${url}`);
                // enable networks events in this target, you can't use the regular API
                // here, so you have to manually build the messages and take care of
                // ids; it's not really a big deal unless you need to match commands and
                // responses
                Target.sendMessageToTarget({
                    sessionId,
                    message: JSON.stringify({ // <-- a JSON string!
                        id: 1,
                        method: 'Network.enable'
                    })
                });
                Target.sendMessageToTarget({
                    sessionId,
                    message: JSON.stringify({ // <-- a JSON string!
                        id: 2,
                        method: 'Target.setAutoAttach',
                        params: {autoAttach: true, waitForDebuggerOnStart:false}
                    })
                });
            });

            Target.receivedMessageFromTarget(({sessionId, message}) => {
                const {id, method, params} = JSON.parse(message); // <-- a JSON string!
                // you can handle massages from the target here; the same apply about
                // the fact that you can't use the regular API here
                // console.log(message);
                switch (method) {
                case 'Network.responseReceived':
                    {
                     
                        //params is the data you need
                        break;
                    }
                case "Target.attachedToTarget":{
                    const childSessionId = params.sessionId;
                    console.log("ATTACHED : " + childSessionId);
                    Target.sendMessageToTarget({
                        sessionId,
                        message: JSON.stringify({
                            id:3,
                            method: 'Target.sendMessageToTarget',
                            params: {
                                sessionId: childSessionId,
                                message: JSON.stringify({                   
                                    id:1,
                                    method:'Network.enable'  
                                })
                            }
                        })
                    });
                    break;
                }
                case "Target.receivedMessageFromTarget": {
                    const childMsg = JSON.parse(params.message);
                    // console.log(childMsg.method);
                    switch (childMsg.method){
                        case "Network.responseReceived" :{
                            //childMsg.params is the data you need
                            break;
                        }
                    }
                    break;
                }
                default:
                    // skip...
                }
            });