no way to change default read_timeout value

I’ve tried the following:

import os
import s3fs

fs = s3fs.S3FileSystem()  #also tried passing use_ssl=False here but no love
# also tried like this where I pass read_timeout here without luck:  s3fs.S3FileSystem(config_kwargs={'read_timeout':3600})

for f in files:
    print(f'Copying bucket1/{f} to bucket2/{f}')
    fs.copy(f'bucket1/{f}', f'bucket2/{f}', config={'read_timeout':3600})
    print('Done', flush=True)

Get the following error:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
~/notebook-env/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, **httplib_request_kw)
    371             try:  # Python 2.7, use buffering of HTTP responses
--> 372                 httplib_response = conn.getresponse(buffering=True)
    373             except TypeError:  # Python 2.6 and older

TypeError: getresponse() got an unexpected keyword argument 'buffering'

During handling of the above exception, another exception occurred:

timeout                                   Traceback (most recent call last)
~/notebook-env/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, **httplib_request_kw)
    373             except TypeError:  # Python 2.6 and older
--> 374                 httplib_response = conn.getresponse()
    375         except (SocketTimeout, BaseSSLError, SocketError) as e:

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py in getresponse(self)
   1330             try:
-> 1331                 response.begin()
   1332             except ConnectionError:

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py in begin(self)
    296         while True:
--> 297             version, status, reason = self._read_status()
    298             if status != CONTINUE:

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/http/client.py in _read_status(self)
    257     def _read_status(self):
--> 258         line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
    259         if len(line) > _MAXLINE:

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/socket.py in readinto(self, b)
    585             try:
--> 586                 return self._sock.recv_into(b)
    587             except timeout:

timeout: timed out

During handling of the above exception, another exception occurred:

ReadTimeoutError                          Traceback (most recent call last)
~/notebook-env/lib/python3.6/site-packages/botocore/vendored/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    369                     retries=self.max_retries,
--> 370                     timeout=timeout
    371                 )

~/notebook-env/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, **response_kw)
    596             retries = retries.increment(method, url, error=e, _pool=self,
--> 597                                         _stacktrace=sys.exc_info()[2])
    598             retries.sleep()

~/notebook-env/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
    244             if read is False:
--> 245                 raise six.reraise(type(error), error, _stacktrace)
    246             elif read is not None:

~/notebook-env/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/packages/six.py in reraise(tp, value, tb)
    309             raise value.with_traceback(tb)
--> 310         raise value
    311 

~/notebook-env/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, **response_kw)
    543                                                   timeout=timeout_obj,
--> 544                                                   body=body, headers=headers)
    545 

~/notebook-env/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, **httplib_request_kw)
    375         except (SocketTimeout, BaseSSLError, SocketError) as e:
--> 376             self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
    377             raise

~/notebook-env/lib/python3.6/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py in _raise_timeout(self, err, url, timeout_value)
    303         if isinstance(err, SocketTimeout):
--> 304             raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value)
    305 

ReadTimeoutError: HTTPConnectionPool(host='s3.amazonaws.com', port=80): Read timed out. (read timeout=15)

During handling of the above exception, another exception occurred:

ReadTimeout                               Traceback (most recent call last)
<ipython-input-3-0844ed2d9b82> in <module>()
      3 for f in files:
      4     print(f'bucket1/{f} to bucket2/{f}')
----> 5     fs.copy(f'bucket1/{f}', f'bucket2/{f}')
      6     print('Done', flush=True)

~/notebook-env/lib/python3.6/site-packages/s3fs/core.py in copy(self, path1, path2, **kwargs)
    686                 self.s3.copy_object,
    687                 kwargs,
--> 688                 Bucket=buc2, Key=key2, CopySource='/'.join([buc1, key1])
    689                 )
    690         except (ClientError, ParamValidationError):

~/notebook-env/lib/python3.6/site-packages/s3fs/core.py in _call_s3(self, method, *akwarglist, **kwargs)
    166         # Add the normal kwargs in
    167         additional_kwargs.update(kwargs)
--> 168         return method(**additional_kwargs)
    169 
    170     @classmethod

~/notebook-env/lib/python3.6/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
    308                     "%s() only accepts keyword arguments." % py_operation_name)
    309             # The "self" in this scope is referring to the BaseClient.
--> 310             return self._make_api_call(operation_name, kwargs)
    311 
    312         _api_call.__name__ = str(py_operation_name)

~/notebook-env/lib/python3.6/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
    584         else:
    585             http, parsed_response = self._endpoint.make_request(
--> 586                 operation_model, request_dict)
    587 
    588         self.meta.events.emit(

~/notebook-env/lib/python3.6/site-packages/botocore/endpoint.py in make_request(self, operation_model, request_dict)
    139         logger.debug("Making request for %s (verify_ssl=%s) with params: %s",
    140                      operation_model, self.verify, request_dict)
--> 141         return self._send_request(request_dict, operation_model)
    142 
    143     def create_request(self, params, operation_model=None):

~/notebook-env/lib/python3.6/site-packages/botocore/endpoint.py in _send_request(self, request_dict, operation_model)
    168             request, operation_model, attempts)
    169         while self._needs_retry(attempts, operation_model, request_dict,
--> 170                                 success_response, exception):
    171             attempts += 1
    172             # If there is a stream associated with the request, we need

~/notebook-env/lib/python3.6/site-packages/botocore/endpoint.py in _needs_retry(self, attempts, operation_model, request_dict, response, caught_exception)
    247             event_name, response=response, endpoint=self,
    248             operation=operation_model, attempts=attempts,
--> 249             caught_exception=caught_exception, request_dict=request_dict)
    250         handler_response = first_non_none_response(responses)
    251         if handler_response is None:

~/notebook-env/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
    225                  handlers.
    226         """
--> 227         return self._emit(event_name, kwargs)
    228 
    229     def emit_until_response(self, event_name, **kwargs):

~/notebook-env/lib/python3.6/site-packages/botocore/hooks.py in _emit(self, event_name, kwargs, stop_on_response)
    208         for handler in handlers_to_call:
    209             logger.debug('Event %s: calling handler %s', event_name, handler)
--> 210             response = handler(**kwargs)
    211             responses.append((handler, response))
    212             if stop_on_response and response is not None:

~/notebook-env/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempts, response, caught_exception, **kwargs)
    181 
    182         """
--> 183         if self._checker(attempts, response, caught_exception):
    184             result = self._action(attempts=attempts)
    185             logger.debug("Retry needed, action of: %s", result)

~/notebook-env/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
    249     def __call__(self, attempt_number, response, caught_exception):
    250         should_retry = self._should_retry(attempt_number, response,
--> 251                                           caught_exception)
    252         if should_retry:
    253             if attempt_number >= self._max_attempts:

~/notebook-env/lib/python3.6/site-packages/botocore/retryhandler.py in _should_retry(self, attempt_number, response, caught_exception)
    275             # If we've exceeded the max attempts we just let the exception
    276             # propogate if one has occurred.
--> 277             return self._checker(attempt_number, response, caught_exception)
    278 
    279 

~/notebook-env/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
    315         for checker in self._checkers:
    316             checker_response = checker(attempt_number, response,
--> 317                                        caught_exception)
    318             if checker_response:
    319                 return checker_response

~/notebook-env/lib/python3.6/site-packages/botocore/retryhandler.py in __call__(self, attempt_number, response, caught_exception)
    221         elif caught_exception is not None:
    222             return self._check_caught_exception(
--> 223                 attempt_number, caught_exception)
    224         else:
    225             raise ValueError("Both response and caught_exception are None.")

~/notebook-env/lib/python3.6/site-packages/botocore/retryhandler.py in _check_caught_exception(self, attempt_number, caught_exception)
    357         # the MaxAttemptsDecorator is not interested in retrying the exception
    358         # then this exception just propogates out past the retry code.
--> 359         raise caught_exception

~/notebook-env/lib/python3.6/site-packages/botocore/endpoint.py in _get_response(self, request, operation_model, attempts)
    202                 request, verify=self.verify,
    203                 stream=operation_model.has_streaming_output,
--> 204                 proxies=self.proxies, timeout=self.timeout)
    205         except ConnectionError as e:
    206             # For a connection error, if it looks like it's a DNS

~/notebook-env/lib/python3.6/site-packages/botocore/vendored/requests/sessions.py in send(self, request, **kwargs)
    571 
    572         # Send the request
--> 573         r = adapter.send(request, **kwargs)
    574 
    575         # Total elapsed time of the request (approximately)

~/notebook-env/lib/python3.6/site-packages/botocore/vendored/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    431                 raise SSLError(e, request=request)
    432             elif isinstance(e, ReadTimeoutError):
--> 433                 raise ReadTimeout(e, request=request)
    434             else:
    435                 raise

ReadTimeout: HTTPConnectionPool(host='s3.amazonaws.com', port=80): Read timed out. (read timeout=15)

It seem like however I pass the read_timeout it doesn’t take effect. Just to verify I could make the error go away by changing the read_timeout value I went ahead and modified the code of core.py (line 205 and 210) to hardcode a longer read_timeout as follows:

if self.anon:
                from botocore import UNSIGNED
                conf = Config(connect_timeout=self.connect_timeout,
                              read_timeout=3600,
                              signature_version=UNSIGNED, **self.config_kwargs)
                self.session = boto3.Session(**self.kwargs)
            else:
                conf = Config(connect_timeout=self.connect_timeout,
                              read_timeout=3600,
                              **self.config_kwargs)

And doing this did fix the timeout error.

[Vslira]

Just mentioning if someone else has this problem: following martindurant’s answer I could fix it, but only by changing the class attribute before instantiating a S3FileSystem object. Trying to set read_timeout = 3600 to an already created object didn’t work

[martindurant]

Currently, the two timeouts are class attributes only. You could set them on the class:

s3fs.S3FileSystem.read_timeout = 3600

or on an instance

 s3 = s3fs.S3FileSystem(...)
 s3.read_timeout = 3600

It would be reasonable to accept these timeouts as keywords to __init__ or as keys in config_kwargs.