@grpc/grpc-js: tryShutdown freeze app

Problem description

If client crashes or exits before server calls call.end(), server.tryShutdown(callback) freezes app. Server has http2session which is closed, so maybeCallback doesn’t decrement pendingChecks, and callback is never called (if tryShutdown was promisified it never resolves).

Reproduction steps

Create server-side stream (hellostreamingworld.proto), end client app before server calls call.end(). (client - greater_client.js without changes)

const promisify = require("util").promisify;
const log = require('why-is-node-running');
var messages = require('./hellostreamingworld_pb');
var services = require('./hellostreamingworld_grpc_pb');
var grpc = require('@grpc/grpc-js');

function sayHello(call) {
  var num = call.request.getNumGreetings();
  console.log(call.request.toObject());
  console.log(`${num} greetings`);
  function greet() {
        if (num-- < 0) {
  		return;
  	}
	var reply = new messages.HelloReply();
	reply.setMessage('Hello ' + call.request.getName());
    call.write(reply);
    setTimeout(greet, 2000);
  }
  setTimeout(greet, 1000);
  call.end();
}

async function main() {
	var server = new grpc.Server();
	server.addService(services.MultiGreeterService, {sayHello: sayHello});
	await promisify(server.bindAsync.bind(server))('0.0.0.0:50051', grpc.ServerCredentials.createInsecure());
	server.start();
	setTimeout(() => {
		console.log('start server shutdown');
		setTimeout(log, 10000);
		server.tryShutdown(() => {
			console.log("server shutdown");
		});
	}, 10000);
}

main();

Environment

• OS name, version and architecture: ArchLinux x86_64
• Node version: v12.18.0
• Node installation method: nvm
• Package name and version [e.g. “@grpc/grpc-js”: “^0.6.16”]

Additional information

server output:

{ name: 'world', numGreetings: 10 }
10 greetings
start server shutdown
There are 8 handle(s) keeping the process running

# TTYWRAP
/home/maxim/code/grpc/examples/node/node_modules/grpc/node_modules/set-blocking/index.js:2 - [process.stdout, process.stderr].forEach(function (stream) {
/home/maxim/code/grpc/examples/node/node_modules/grpc/node_modules/npmlog/log.js:11        - setBlocking(true)

# SIGNALWRAP
/home/maxim/code/grpc/examples/node/node_modules/grpc/node_modules/set-blocking/index.js:2 - [process.stdout, process.stderr].forEach(function (stream) {
/home/maxim/code/grpc/examples/node/node_modules/grpc/node_modules/npmlog/log.js:11        - setBlocking(true)

# TTYWRAP
/home/maxim/code/grpc/examples/node/node_modules/grpc/node_modules/set-blocking/index.js:2 - [process.stdout, process.stderr].forEach(function (stream) {
/home/maxim/code/grpc/examples/node/node_modules/grpc/node_modules/npmlog/log.js:11        - setBlocking(true)

# DNSCHANNEL
(unknown stack trace)

# HTTP2SESSION
(unknown stack trace)

# HTTP2SETTINGS
(unknown stack trace)

# Timeout
/home/maxim/code/grpc/examples/node/static_codegen/greeter_server.js:55 - setTimeout(log, 10000);

# Timeout
/home/maxim/code/grpc/examples/node/static_codegen/greeter_server.js:38 - setTimeout(greet, 2000);

[CMCDragonkai]

I’m facing a similar problem, but tryShutdown doesn’t ever finish when the only thing I’ve done is to create a duplex stream, and then end it from both sides. Both sides report that the stream is destroyed, the client is closed, but then the server continues to wait to shutdown. I have to force shutdown. Seems the tryShutdown is a bit flaky?

[murgatroid99]

I was able to get some logs by setting the environment variables GRPC_TRACE=all and GRPC_VERBOSITY=DEBUG when running that test. According to those logs, the client opens two streams, and the second one finishes, but the first one never does. It is intended behavior that the server does not call the callback for tryShutdown until all streams have finished. If you want to force the server to shut down you should call forceShutdown.

In the PR you linked, you are awaiting the result of tryShutdown on line 184. The code will not progress past that point until the server has finished shutting down, so it never actually calls forceShutdown on line 191. If you remove the call to await on line 184 that will probably fix it.