HTTP: WebSocket crash on WsPerformingCheckState leads to "Unexpected onTimeout call in WsCrashedState"

Version: 3.5.1

In some situations our simulations have a few (one or two) users hanging due to a crashed websocket connection. I would expect that the crashed websocket does not lead to some internal timeouts still being triggered and the users hanging indefinitely.

After playing around with a simple simulation and a websocket server implementation in NodeJS I was able to reproduce this when the WebSocket server itself crashes after the connection.

Gatling Scenario:

import io.gatling.core.Predef._
import io.gatling.http.Predef._
import scala.concurrent.duration._
import io.gatling.http.protocol.HttpProtocolBuilder

class DevSim extends Simulation {

  def httpConf: HttpProtocolBuilder = {
    var httpConf = http
      .baseUrl("https://localhost")
      .acceptHeader("application/json, application/hal+json")
      .acceptLanguageHeader("en-US,en;q=0.5")
      .acceptEncodingHeader("gzip, deflate")
      .userAgentHeader("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36")
      .wsBaseUrl("ws://localhost:81")
      .inferHtmlResources()
      .disableWarmUp
    httpConf
  }

  def webSocketName = "testWebSocket"

  def devSim = scenario("WS Test Scenario")
    .exitBlockOnFail(
      exec(ws("Connect").wsName(webSocketName).connect("ws://localhost:81").await(1.second)(ws.checkTextMessage("login").silent))
    )
    .repeat(10) {
      exitBlockOnFail(
      exec(ws("Send Text").wsName(webSocketName).sendText("Hello")
        .await(1.second)(
          ws.checkTextMessage("Check Result").check(regex("Hi, you sent me Hello").exists)
        ))
        .pause(5.second)
      )
    }

  setUp(
    devSim.inject(atOnceUsers(1))
      .protocols(httpConf)
  ).assertions(global.successfulRequests.percent.is(100))
}

WebSocket Server:

var WebSocket = require('ws');

const ws_server = new WebSocket.Server({ port: 81 });
function log(message) {
    console.log(new Date(Date.now()) + " - " + message)
}

ws_server.on('connection', function connection(ws) {
    // important is that the JSON.stringify(ws) call let's the server crash
   // because it is trying to convert a circular structure to JSON
    log("A client connected - " + JSON.stringify(ws));
    ws.on('message', function incoming(message) {
        log("Received " + message);
        ws.send('Hi, you sent me ' + message);
    });
});

ws_server.on('close', function () {
    log("A client closed the connection.");
});

Log output:

5269 [GatlingSystem-akka.actor.default-dispatcher-6] INFO  i.g.core.controller.inject.Injector - Injecting is done
5312 [gatling-1-1] DEBUG io.gatling.http.action.ws.WsConnect - Opening websocket 'testWebSocket': Scenario 'WS Test Scenario', UserId #1
5312 [gatling-1-1] DEBUG io.gatling.http.action.ws.WsConnect - Opening websocket 'testWebSocket': Scenario 'WS Test Scenario', UserId #1
5323 [gatling-1-1] DEBUG i.g.h.a.ws.fsm.WsConnectingState$ - Connecting to ws://localhost:81
5323 [gatling-1-1] DEBUG i.g.h.a.ws.fsm.WsConnectingState$ - Connecting to ws://localhost:81
[...]
9093 [gatling-1-1] DEBUG i.g.c.c.inject.open.OpenWorkload - Start user #1
9121 [gatling-1-1] DEBUG i.n.h.c.h.w.WebSocketClientHandshaker13 - WebSocket version 13 client handshake key: cihrOLzJIaAy/IAw2fAj6A==, expected response: ScedZaO/3T1F5R3OthdFOG9EtdE=
[...]
9161 [gatling-1-1] DEBUG i.gatling.http.action.ws.WsListener - Received response to WebSocket CONNECT: 101 Switching Protocols DefaultHttpHeaders[Upgrade: websocket, Connection: Upgrade, Sec-WebSocket-Accept: ScedZaO/3T1F5R3OthdFOG9EtdE=, content-length: 0]
9161 [gatling-1-1] DEBUG i.gatling.http.action.ws.WsListener - Received response to WebSocket CONNECT: 101 Switching Protocols DefaultHttpHeaders[Upgrade: websocket, Connection: Upgrade, Sec-WebSocket-Accept: ScedZaO/3T1F5R3OthdFOG9EtdE=, content-length: 0]
9165 [gatling-1-1] DEBUG i.g.h.a.ws.fsm.WsConnectingState - Connected, performing checks before proceeding (no onConnected action)
9165 [gatling-1-1] DEBUG i.g.h.a.ws.fsm.WsConnectingState - Connected, performing checks before proceeding (no onConnected action)
9167 [gatling-1-1] DEBUG io.gatling.http.action.ws.fsm.WsFsm - Timeout 61592924 scheduled
9167 [gatling-1-1] DEBUG io.gatling.http.action.ws.fsm.WsFsm - Timeout 61592924 scheduled
9247 [gatling-1-1] DEBUG i.g.h.client.impl.WebSocketHandler - channelInactive
9247 [gatling-1-1] DEBUG i.g.h.client.impl.WebSocketHandler - channelInactive
9250 [gatling-1-1] DEBUG i.g.h.client.impl.WebSocketHandler - Crash
io.gatling.http.client.impl.HttpAppHandler$1: Premature close
9250 [gatling-1-1] DEBUG i.g.h.client.impl.WebSocketHandler - Crash
io.gatling.http.client.impl.HttpAppHandler$1: Premature close
9254 [gatling-1-1] DEBUG i.g.h.a.w.f.WsPerformingCheckState - WebSocket crashed by the server while in WsPerformingCheckState state
io.gatling.http.client.impl.HttpAppHandler$1: Premature close
9254 [gatling-1-1] DEBUG i.g.h.a.w.f.WsPerformingCheckState - WebSocket crashed by the server while in WsPerformingCheckState state
io.gatling.http.client.impl.HttpAppHandler$1: Premature close
10176 [gatling-1-1] DEBUG io.gatling.http.action.ws.fsm.WsFsm - Timeout 61592924 triggered
10176 [gatling-1-1] DEBUG io.gatling.http.action.ws.fsm.WsFsm - Timeout 61592924 triggered
10177 [gatling-1-1] ERROR i.g.h.action.ws.fsm.WsCrashedState - Unexpected onTimeout call in WsCrashedState state
java.lang.IllegalStateException: Unexpected onTimeout call in WsCrashedState state
	at io.gatling.http.action.ws.fsm.WsState.onIllegalState(WsState.scala:80)
	at io.gatling.http.action.ws.fsm.WsState.onTimeout(WsState.scala:77)
	at io.gatling.http.action.ws.fsm.WsFsm.$anonfun$scheduleTimeout$2(WsFsm.scala:56)
	at io.gatling.http.action.ws.fsm.WsFsm.execute(WsFsm.scala:77)
	at io.gatling.http.action.ws.fsm.WsFsm.$anonfun$scheduleTimeout$1(WsFsm.scala:56)
	at io.gatling.http.action.ws.fsm.WsFsm.$anonfun$scheduleTimeout$1$adapted(WsFsm.scala:53)
	at io.netty.util.concurrent.PromiseTask.runTask(PromiseTask.java:96)
	at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:170)
	at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)
	at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.lang.Thread.run(Thread.java:748)
10177 [gatling-1-1] ERROR i.g.h.action.ws.fsm.WsCrashedState - Unexpected onTimeout call in WsCrashedState state
java.lang.IllegalStateException: Unexpected onTimeout call in WsCrashedState state
	at io.gatling.http.action.ws.fsm.WsState.onIllegalState(WsState.scala:80)
	at io.gatling.http.action.ws.fsm.WsState.onTimeout(WsState.scala:77)
	at io.gatling.http.action.ws.fsm.WsFsm.$anonfun$scheduleTimeout$2(WsFsm.scala:56)
	at io.gatling.http.action.ws.fsm.WsFsm.execute(WsFsm.scala:77)
	at io.gatling.http.action.ws.fsm.WsFsm.$anonfun$scheduleTimeout$1(WsFsm.scala:56)
	at io.gatling.http.action.ws.fsm.WsFsm.$anonfun$scheduleTimeout$1$adapted(WsFsm.scala:53)
	at io.netty.util.concurrent.PromiseTask.runTask(PromiseTask.java:96)
	at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:170)
	at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)
	at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.lang.Thread.run(Thread.java:748)
10179 [gatling-1-1] DEBUG i.g.h.action.ws.fsm.WsCrashedState - WebSocket crashed by the server while in WsCrashedState state
java.lang.IllegalStateException: Unexpected onTimeout call in WsCrashedState state
	at io.gatling.http.action.ws.fsm.WsState.onIllegalState(WsState.scala:80)
	at io.gatling.http.action.ws.fsm.WsState.onTimeout(WsState.scala:77)
	at io.gatling.http.action.ws.fsm.WsFsm.$anonfun$scheduleTimeout$2(WsFsm.scala:56)
	at io.gatling.http.action.ws.fsm.WsFsm.execute(WsFsm.scala:77)
	at io.gatling.http.action.ws.fsm.WsFsm.$anonfun$scheduleTimeout$1(WsFsm.scala:56)
	at io.gatling.http.action.ws.fsm.WsFsm.$anonfun$scheduleTimeout$1$adapted(WsFsm.scala:53)
	at io.netty.util.concurrent.PromiseTask.runTask(PromiseTask.java:96)
	at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:170)
	at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)
	at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.lang.Thread.run(Thread.java:748)
10179 [gatling-1-1] DEBUG i.g.h.action.ws.fsm.WsCrashedState - WebSocket crashed by the server while in WsCrashedState state
java.lang.IllegalStateException: Unexpected onTimeout call in WsCrashedState state
	at io.gatling.http.action.ws.fsm.WsState.onIllegalState(WsState.scala:80)
	at io.gatling.http.action.ws.fsm.WsState.onTimeout(WsState.scala:77)
	at io.gatling.http.action.ws.fsm.WsFsm.$anonfun$scheduleTimeout$2(WsFsm.scala:56)
	at io.gatling.http.action.ws.fsm.WsFsm.execute(WsFsm.scala:77)
	at io.gatling.http.action.ws.fsm.WsFsm.$anonfun$scheduleTimeout$1(WsFsm.scala:56)
	at io.gatling.http.action.ws.fsm.WsFsm.$anonfun$scheduleTimeout$1$adapted(WsFsm.scala:53)
	at io.netty.util.concurrent.PromiseTask.runTask(PromiseTask.java:96)
	at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:170)
	at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)
	at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)
	at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500)
	at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)
	at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.lang.Thread.run(Thread.java:748)

Full log output can be found here

[slandelle]

Fixed, thanks for reporting again!

[slandelle]

@jklingelAvid Lack of sleep, sorry 😃

Thanks, having a look.