Connection Issue - Reset by Peer

The same code on the same Ec2 instance now throw the error below. It appears to come from the line in my code: register_trainable(“exp”, obj_function)

I have redis ‘2.10.6’ installed.

Any hints on the cause?

Process STDOUT and STDERR is being redirected to /tmp/raylogs/.
Waiting for redis server at 127.0.0.1:60723 to respond...
Waiting for redis server at 127.0.0.1:48080 to respond...
Starting local scheduler with the following resources: {'CPU': 96, 'GPU': 0}.

======================================================================
View the web UI at http://localhost:8889/notebooks/ray_ui33174.ipynb?token=74b2c0738a07e667d60580a4b2f24d7317954eaf830ab0ef
======================================================================

---------------------------------------------------------------------------
ConnectionResetError                      Traceback (most recent call last)
~/anaconda3/lib/python3.6/site-packages/redis/connection.py in send_packed_command(self, command)
    589             for item in command:
--> 590                 self._sock.sendall(item)
    591         except socket.timeout:

ConnectionResetError: [Errno 104] Connection reset by peer

During handling of the above exception, another exception occurred:

ConnectionError                           Traceback (most recent call last)
~/anaconda3/lib/python3.6/site-packages/redis/client.py in execute_command(self, *args, **options)
    666         try:
--> 667             connection.send_command(*args)
    668             return self.parse_response(connection, command_name, **options)

~/anaconda3/lib/python3.6/site-packages/redis/connection.py in send_command(self, *args)
    609         "Pack and send a command to the Redis server"
--> 610         self.send_packed_command(self.pack_command(*args))
    611 

~/anaconda3/lib/python3.6/site-packages/redis/connection.py in send_packed_command(self, command)
    602             raise ConnectionError("Error %s while writing to socket. %s." %
--> 603                                   (errno, errmsg))
    604         except:

ConnectionError: Error 104 while writing to socket. Connection reset by peer.

During handling of the above exception, another exception occurred:

ConnectionResetError                      Traceback (most recent call last)
~/anaconda3/lib/python3.6/site-packages/redis/connection.py in send_packed_command(self, command)
    589             for item in command:
--> 590                 self._sock.sendall(item)
    591         except socket.timeout:

ConnectionResetError: [Errno 104] Connection reset by peer

During handling of the above exception, another exception occurred:

ConnectionError                           Traceback (most recent call last)
<ipython-input-6-48c88d974e24> in <module>()
     75         ray.init()
     76 
---> 77     register_trainable("exp", obj_function) #registers the above config and the objective function
     78 
     79     hpo=HyperOptSearch(space, max_concurrent=4, reward_attr="neg_mean_loss") #smaller is better for log loss

~/anaconda3/lib/python3.6/site-packages/ray/tune/registry.py in register_trainable(name, trainable)
     36         raise TypeError("Second argument must be convertable to Trainable",
     37                         trainable)
---> 38     _global_registry.register(TRAINABLE_CLASS, name, trainable)
     39 
     40 

~/anaconda3/lib/python3.6/site-packages/ray/tune/registry.py in register(self, category, key, value)
     77         self._to_flush[(category, key)] = pickle.dumps(value)
     78         if _internal_kv_initialized():
---> 79             self.flush_values()
     80 
     81     def contains(self, category, key):

~/anaconda3/lib/python3.6/site-packages/ray/tune/registry.py in flush_values(self)
     99     def flush_values(self):
    100         for (category, key), value in self._to_flush.items():
--> 101             _internal_kv_put(_make_key(category, key), value, overwrite=True)
    102         self._to_flush.clear()
    103 

~/anaconda3/lib/python3.6/site-packages/ray/experimental/internal_kv.py in _internal_kv_put(key, value, overwrite)
     29     worker = ray.worker.get_global_worker()
     30     if overwrite:
---> 31         updated = worker.redis_client.hset(key, "value", value)
     32     else:
     33         updated = worker.redis_client.hsetnx(key, "value", value)

~/anaconda3/lib/python3.6/site-packages/ray/utils.py in _wrapper(*args, **kwargs)
    322                 def _wrapper(*args, **kwargs):
    323                     with self.lock:
--> 324                         return orig_attr(*args, **kwargs)
    325 
    326                 self._wrapper_cache[attr] = _wrapper

~/anaconda3/lib/python3.6/site-packages/redis/client.py in hset(self, name, key, value)
   1990         Returns 1 if HSET created a new field, otherwise 0
   1991         """
-> 1992         return self.execute_command('HSET', name, key, value)
   1993 
   1994     def hsetnx(self, name, key, value):

~/anaconda3/lib/python3.6/site-packages/redis/client.py in execute_command(self, *args, **options)
    671             if not connection.retry_on_timeout and isinstance(e, TimeoutError):
    672                 raise
--> 673             connection.send_command(*args)
    674             return self.parse_response(connection, command_name, **options)
    675         finally:

~/anaconda3/lib/python3.6/site-packages/redis/connection.py in send_command(self, *args)
    608     def send_command(self, *args):
    609         "Pack and send a command to the Redis server"
--> 610         self.send_packed_command(self.pack_command(*args))
    611 
    612     def can_read(self, timeout=0):

~/anaconda3/lib/python3.6/site-packages/redis/connection.py in send_packed_command(self, command)
    601                 errmsg = e.args[1]
    602             raise ConnectionError("Error %s while writing to socket. %s." %
--> 603                                   (errno, errmsg))
    604         except:
    605             self.disconnect()

ConnectionError: Error 104 while writing to socket. Connection reset by peer.

[flowersw]

How big is your data set? I kept getting ConnectionResetError: [Errno 104] Connection reset by peer errors. Then, I followed the guide on working with large objects (mine were in the GB range) here: https://ray.readthedocs.io/en/latest/tune-usage.html#handling-large-datasets and I no longer got that error.

[richardliaw]

How big is your data set? I kept getting ConnectionResetError: [Errno 104] Connection reset by peer errors. Then, I followed the guide on working with large objects (mine were in the GB range) here: ray.readthedocs.io/en/latest/tune-usage.html#handling-large-datasets and I no longer got that error.

This is now located at https://docs.ray.io/en/latest/tune/tutorials/tune-usage.html#handling-large-datasets