Errors after responses update to 0.19.0

Hey,

First off, thanks for the work you input on this very useful project.

I am seeing a new issue when using moto[s3] in my unit tests. Here is the snippet of code I use:

import boto3
import os
import pytest
from moto import mock_s3

from s3management import get_mapping_file


bucket = "test-bucket"
mappinglocation = "es/devstatus.mapping"

@pytest.fixture(scope='function')
def aws_credentials():
    """Mocked AWS Credentials for moto."""
    os.environ['AWS_ACCESS_KEY_ID'] = 'testing'
    os.environ['AWS_SECRET_ACCESS_KEY'] = 'testing'
    os.environ['AWS_SECURITY_TOKEN'] = 'testing'
    os.environ['AWS_SESSION_TOKEN'] = 'testing'
    os.environ['AWS_DEFAULT_REGION'] = 'us-east-1'


@pytest.fixture(scope='function')
def s3(aws_credentials):
    with mock_s3():
        yield boto3.resource('s3', region_name='us-east-1')


@mock_s3
def test_get_mapping(s3):
    dataset = "devstatus"
    s3path = "s3://test-bucket/es/"

    s3.create_bucket(Bucket=bucket)
    obj = s3.Object(bucket, mappinglocation).put()

    result = get_mapping_file(s3path, dataset)
    expected_result = (f"{dataset}.mapping", bucket)

    assert result == expected_result

Since, I suspect, today’s update in responses module from 0.18.0 to 0.19.0, I am getting following errors:

=================================== FAILURES ===================================
_______________________________ test_get_mapping _______________________________

args = (), kwargs = {'s3': s3.ServiceResource()}

    def wrapper(*args, **kwargs):
>       self.start(reset=reset)

/usr/local/lib/python3.9/site-packages/moto/core/models.py:118: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/usr/local/lib/python3.9/site-packages/moto/core/models.py:96: in start
    self.enable_patching(reset)
/usr/local/lib/python3.9/site-packages/moto/core/models.py:358: in enable_patching
    responses_mock.add(
/usr/local/lib/python3.9/site-packages/responses/__init__.py:603: in add
    self._registry.add(method)
/usr/local/lib/python3.9/site-packages/responses/registries.py:53: in add
    response = copy.deepcopy(response)
/usr/local/lib/python3.9/copy.py:172: in deepcopy
    y = _reconstruct(x, memo, *rv)
/usr/local/lib/python3.9/copy.py:270: in _reconstruct
    state = deepcopy(state, memo)
/usr/local/lib/python3.9/copy.py:146: in deepcopy
    y = copier(x, memo)
/usr/local/lib/python3.9/copy.py:230: in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
/usr/local/lib/python3.9/copy.py:172: in deepcopy
    y = _reconstruct(x, memo, *rv)
/usr/local/lib/python3.9/copy.py:270: in _reconstruct
    state = deepcopy(state, memo)
/usr/local/lib/python3.9/copy.py:146: in deepcopy
    y = copier(x, memo)
/usr/local/lib/python3.9/copy.py:230: in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
/usr/local/lib/python3.9/copy.py:146: in deepcopy
    y = copier(x, memo)
/usr/local/lib/python3.9/copy.py:237: in _deepcopy_method
    return type(x)(x.__func__, deepcopy(x.__self__, memo))
/usr/local/lib/python3.9/copy.py:172: in deepcopy
    y = _reconstruct(x, memo, *rv)
/usr/local/lib/python3.9/copy.py:270: in _reconstruct
    state = deepcopy(state, memo)
/usr/local/lib/python3.9/copy.py:146: in deepcopy
    y = copier(x, memo)
/usr/local/lib/python3.9/copy.py:230: in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
/usr/local/lib/python3.9/copy.py:172: in deepcopy
    y = _reconstruct(x, memo, *rv)
/usr/local/lib/python3.9/copy.py:270: in _reconstruct
    state = deepcopy(state, memo)
/usr/local/lib/python3.9/copy.py:146: in deepcopy
    y = copier(x, memo)
/usr/local/lib/python3.9/copy.py:230: in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
/usr/local/lib/python3.9/copy.py:172: in deepcopy
    y = _reconstruct(x, memo, *rv)
/usr/local/lib/python3.9/copy.py:270: in _reconstruct
    state = deepcopy(state, memo)
/usr/local/lib/python3.9/copy.py:146: in deepcopy
    y = copier(x, memo)
/usr/local/lib/python3.9/copy.py:230: in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
/usr/local/lib/python3.9/copy.py:146: in deepcopy
    y = copier(x, memo)
/usr/local/lib/python3.9/copy.py:230: in _deepcopy_dict
    y[deepcopy(key, memo)] = deepcopy(value, memo)
/usr/local/lib/python3.9/copy.py:172: in deepcopy
    y = _reconstruct(x, memo, *rv)
/usr/local/lib/python3.9/copy.py:264: in _reconstruct
    y = func(*args)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

cls = <class 'jinja2.environment.Template'>, args = ()

    def __newobj__(cls, *args):
>       return cls.__new__(cls, *args)
E       TypeError: __new__() missing 1 required positional argument: 'source'

/usr/local/lib/python3.9/copyreg.py:95: TypeError

Versions I use are following: awscli==1.19.112 moto[s3]==3.0.4 pytest==7.0.1

If I add responses==0.18.0 this error doesn’t occur. Not sure exactly what is it related to, mayne the usage of get_response_mock() method…or so it seems.

Let me know if you need any further info or tests on my side. Thanks!

[bblommers]

@tduque-electric Moto 3.0.7 was released just now: https://pypi.org/project/moto/3.0.7/ 👍

[bblommers]

@jwilmott It is on PyPi! https://pypi.org/project/moto/3.0.7.dev2/