boto3 + moto + s3 + Docker: "ValueError, "I/O operation on closed file""

This simple test passes locally (when ran using py.test)

@mock_s3
def test_read_data():
    temp = tempfile.NamedTemporaryFile()
    temp.write('Something')
    temp.seek(0)

    s3 = boto3.resource('s3')
    s3.create_bucket(Bucket='mybucket')
    s3.Object('mybucket', 'hello.txt').put(Body=open(temp.name, 'rb'))

    obj = s3.Object(bucket_name='mybucket', key='hello.txt')
    response = obj.get()
    data = response['Body'].read()

    assert data == 'Something'

However, the same test fails when run inside a Docker container (docker build .):

Traceback (most recent call last):
  File "/usr/local/lib/python2.7/site-packages/botocore/endpoint.py", line 168, in _get_response
    proxies=self.proxies, timeout=self.timeout)
  File "/usr/local/lib/python2.7/site-packages/botocore/vendored/requests/sessions.py", line 573, in send
    r = adapter.send(request, **kwargs)
  File "/usr/local/lib/python2.7/site-packages/botocore/vendored/requests/adapters.py", line 370, in send
    timeout=timeout
  File "/usr/local/lib/python2.7/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py", line 544, in urlopen
    body=body, headers=headers)
  File "/usr/local/lib/python2.7/site-packages/botocore/vendored/requests/packages/urllib3/connectionpool.py", line 372, in _make_request
    httplib_response = conn.getresponse(buffering=True)
  File "/usr/local/lib/python2.7/httplib.py", line 1129, in getresponse
    response = self.response_class(*args, **kwds)
  File "/usr/local/lib/python2.7/site-packages/botocore/awsrequest.py", line 45, in __init__
    HTTPResponse.__init__(self, *args, **kwargs)
  File "/usr/local/lib/python2.7/httplib.py", line 383, in __init__
    self.fp = sock.makefile('rb')
  File "/usr/local/lib/python2.7/site-packages/httpretty/core.py", line 332, in makefile
    self._entry.fill_filekind(self.fd)
  File "/usr/local/lib/python2.7/site-packages/httpretty/core.py", line 621, in fill_filekind
    fk.write(utf8(item) + b'\n')
  File "/usr/local/lib/python2.7/StringIO.py", line 213, in write
    _complain_ifclosed(self.closed)
  File "/usr/local/lib/python2.7/StringIO.py", line 40, in _complain_ifclosed
    raise ValueError, "I/O operation on closed file"
ValueError: I/O operation on closed file

Maybe this issue should be posted to httppretty instead of here.

Source code: boto3_pytest_docker.zip

[ianfdk]

I had a similar issue with StringIO and got rid of it by putting the contents of the file instead of the file itself.

Something like this:

output_str = 'Something'
bucket = s3.Bucket('mybucket')
bucket.put_object(Body=output_str, Key='hello.txt')

[mschneiderwind]

Same issue with moto 1.1.21 (using boto3==1.4.7) Going back to 0.4.x fixes the issue

my code

import os
from pathlib import Path

import boto3


def download_from_s3(keys, download_dir):
    download_dir = Path(download_dir)
    s3 = boto3.resource(service_name='s3')
    bucket = s3.Bucket(os.environ['BUCKET_NAME'])
    files = []
    for key in keys:
        filename = download_dir / key
        filename.parent.mkdir(parents=True, exist_ok=True)
        bucket.download_file(Key=key, Filename=str(filename))
        files += [str(filename)]
    return files


def upload_to_s3(file, key):
    s3 = boto3.resource(service_name='s3')
    bucket = s3.Bucket(os.environ['BUCKET_NAME'])
    bucket.upload_file(Filename=file, Key=key)

failling test:

import os
import tempfile
from pathlib import Path

import boto3
from moto import mock_s3

from mys3 import upload_to_s3, download_from_s3

@mock_s3
def test_s3():
    s3 = boto3.resource(service_name='s3')
    bucket = s3.Bucket(os.environ['BUCKET_NAME'])
    bucket.create()

    with tempfile.NamedTemporaryFile() as tf:
        tf.write(b"hello")
        upload_to_s3(file=tf.name, key="toto/tata/2016.23/a.txt")

    with tempfile.TemporaryDirectory() as tmpdir:
        # files = download_from_s3(keys=res, download_dir=tmpdir)
        download_dir = Path(tmpdir)
        key = "toto/tata/2016.23/a.txt"
        filename = download_dir / key
        filename.parent.mkdir(parents=True, exist_ok=True)
        bucket.download_file(Key=key, Filename=str(filename))

    bucket.objects.all().delete()
    bucket.delete()

trace:

/usr/local/lib/python3.6/site-packages/boto3/s3/inject.py:168: in bucket_download_file
    ExtraArgs=ExtraArgs, Callback=Callback, Config=Config)
/usr/local/lib/python3.6/site-packages/boto3/s3/inject.py:130: in download_file
    extra_args=ExtraArgs, callback=Callback)
/usr/local/lib/python3.6/site-packages/boto3/s3/transfer.py:299: in download_file
    future.result()
/usr/local/lib/python3.6/site-packages/s3transfer/futures.py:73: in result
    return self._coordinator.result()
/usr/local/lib/python3.6/site-packages/s3transfer/futures.py:233: in result
    raise self._exception
/usr/local/lib/python3.6/site-packages/s3transfer/tasks.py:126: in __call__
    return self._execute_main(kwargs)
/usr/local/lib/python3.6/site-packages/s3transfer/tasks.py:150: in _execute_main
    return_value = self._main(**kwargs)
/usr/local/lib/python3.6/site-packages/s3transfer/download.py:518: in _main
    for chunk in chunks:
/usr/local/lib/python3.6/site-packages/s3transfer/download.py:646: in __next__
    chunk = self._body.read(self._chunksize)
/usr/local/lib/python3.6/site-packages/s3transfer/utils.py:491: in read
    value = self._stream.read(*args, **kwargs)
/usr/local/lib/python3.6/site-packages/botocore/response.py:74: in read
    chunk = self._raw_stream.read(amt)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <urllib3.response.HTTPResponse object at 0x7f1200200630>, amt = 262144, decode_content = False, cache_content = False

    def read(self, amt=None, decode_content=None, cache_content=False):
        """
            Similar to :meth:`httplib.HTTPResponse.read`, but with two additional
            parameters: ``decode_content`` and ``cache_content``.
    
            :param amt:
                How much of the content to read. If specified, caching is skipped
                because it doesn't make sense to cache partial content as the full
                response.
    
            :param decode_content:
                If True, will attempt to decode the body based on the
                'content-encoding' header.
    
            :param cache_content:
                If True, will save the returned data such that the same result is
                returned despite of the state of the underlying file object. This
                is useful if you want the ``.data`` property to continue working
                after having ``.read()`` the file object. (Overridden if ``amt`` is
                set.)
            """
        self._init_decoder()
        if decode_content is None:
            decode_content = self.decode_content
    
        if self._fp is None:
            return
    
        flush_decoder = False
        data = None
    
        with self._error_catcher():
            if amt is None:
                # cStringIO doesn't like amt=None
                data = self._fp.read()
                flush_decoder = True
            else:
                cache_content = False
>               data = self._fp.read(amt)
E               ValueError: I/O operation on closed file.

amt        = 262144
cache_content = False
data       = None
decode_content = False
flush_decoder = False
self       = <urllib3.response.HTTPResponse object at 0x7f1200200630>

/usr/local/lib/python3.6/site-packages/urllib3/response.py:384: ValueError