Tavern is intrusive and implicitly breaks test suites of unrelated packages

If tavern is installed on the system (tested with 1.24.1) test suites of other packages are implicitly broken. This is a very bad practice since tavern can be installed as a dependency of one package but at the same time break other packages in unpredictable ways.

For example, the test suite of apispec fails two tests:

_________________________________________ test_load_yaml_from_docstring_empty_docstring[---] __________________________________________

docstring = '---'

    @pytest.mark.parametrize("docstring", (None, "", "---"))
    def test_load_yaml_from_docstring_empty_docstring(docstring):
>       assert yaml_utils.load_yaml_from_docstring(docstring) == {}

docstring  = '---'

tests/test_yaml_utils.py:23: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../apispec-6.0.2-python3_10/install/usr/lib/python3.10/site-packages/apispec/yaml_utils.py:35: in load_yaml_from_docstring
    return yaml.safe_load(yaml_string) or {}
        cut_from   = 0
        docstring  = '---'
        index      = 0
        line       = '---'
        split_lines = ['---']
        yaml_string = '---'
/usr/lib/python3.10/site-packages/yaml/__init__.py:125: in safe_load
    return load(stream, SafeLoader)
        stream     = '---'
/usr/lib/python3.10/site-packages/yaml/__init__.py:81: in load
    return loader.get_single_data()
        Loader     = <class 'yaml.loader.SafeLoader'>
        loader     = <yaml.loader.SafeLoader object at 0x7f86de0573a0>
        stream     = '---'
/usr/lib/python3.10/site-packages/yaml/constructor.py:49: in get_single_data
    node = self.get_single_node()
        self       = <yaml.loader.SafeLoader object at 0x7f86de0573a0>
/usr/lib/python3.10/site-packages/yaml/composer.py:36: in get_single_node
    document = self.compose_document()
        document   = None
        self       = <yaml.loader.SafeLoader object at 0x7f86de0573a0>
/usr/lib/python3.10/site-packages/yaml/composer.py:55: in compose_document
    node = self.compose_node(None, None)
        self       = <yaml.loader.SafeLoader object at 0x7f86de0573a0>
/usr/lib/python3.10/site-packages/yaml/composer.py:64: in compose_node
    if self.check_event(AliasEvent):
        index      = None
        parent     = None
        self       = <yaml.loader.SafeLoader object at 0x7f86de0573a0>
/usr/lib/python3.10/site-packages/yaml/parser.py:98: in check_event
    self.current_event = self.state()
        choices    = (<class 'yaml.events.AliasEvent'>,)
        self       = <yaml.loader.SafeLoader object at 0x7f86de0573a0>
/usr/lib/python3.10/site-packages/yaml/parser.py:211: in parse_document_content
    event = self.process_empty_scalar(self.peek_token().start_mark)
        self       = <yaml.loader.SafeLoader object at 0x7f86de0573a0>
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <yaml.loader.SafeLoader object at 0x7f86de0573a0>, mark = <yaml.error.Mark object at 0x7f86de057520>

    def error_on_empty_scalar(self, mark):  # pylint: disable=unused-argument
        location = "{mark.name:s}:{mark.line:d} - column {mark.column:d}".format(mark=mark)
        error = "Error at {} - cannot define an empty value in test - either give it a value or explicitly set it to None".format(
            location
        )
    
>       raise exceptions.BadSchemaError(error)
E       tavern.util.exceptions.BadSchemaError: Error at <unicode string>:0 - column 3 - cannot define an empty value in test - either give it a value or explicitly set it to None

error      = ('Error at <unicode string>:0 - column 3 - cannot define an empty value in '
 'test - either give it a value or explicitly set it to None')
location   = '<unicode string>:0 - column 3'
mark       = <yaml.error.Mark object at 0x7f86de057520>
self       = <yaml.loader.SafeLoader object at 0x7f86de0573a0>

/usr/lib/python3.10/site-packages/tavern/util/loader.py:455: BadSchemaError
______________________________________ test_load_operations_from_docstring_empty_docstring[---] _______________________________________

docstring = '---'

    @pytest.mark.parametrize("docstring", (None, "", "---"))
    def test_load_operations_from_docstring_empty_docstring(docstring):
>       assert yaml_utils.load_operations_from_docstring(docstring) == {}

docstring  = '---'

tests/test_yaml_utils.py:28: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../apispec-6.0.2-python3_10/install/usr/lib/python3.10/site-packages/apispec/yaml_utils.py:45: in load_operations_from_docstring
    doc_data = load_yaml_from_docstring(docstring)
        docstring  = '---'
../apispec-6.0.2-python3_10/install/usr/lib/python3.10/site-packages/apispec/yaml_utils.py:35: in load_yaml_from_docstring
    return yaml.safe_load(yaml_string) or {}
        cut_from   = 0
        docstring  = '---'
        index      = 0
        line       = '---'
        split_lines = ['---']
        yaml_string = '---'
/usr/lib/python3.10/site-packages/yaml/__init__.py:125: in safe_load
    return load(stream, SafeLoader)
        stream     = '---'
/usr/lib/python3.10/site-packages/yaml/__init__.py:81: in load
    return loader.get_single_data()
        Loader     = <class 'yaml.loader.SafeLoader'>
        loader     = <yaml.loader.SafeLoader object at 0x7f86ddf5b670>
        stream     = '---'
/usr/lib/python3.10/site-packages/yaml/constructor.py:49: in get_single_data
    node = self.get_single_node()
        self       = <yaml.loader.SafeLoader object at 0x7f86ddf5b670>
/usr/lib/python3.10/site-packages/yaml/composer.py:36: in get_single_node
    document = self.compose_document()
        document   = None
        self       = <yaml.loader.SafeLoader object at 0x7f86ddf5b670>
/usr/lib/python3.10/site-packages/yaml/composer.py:55: in compose_document
    node = self.compose_node(None, None)
        self       = <yaml.loader.SafeLoader object at 0x7f86ddf5b670>
/usr/lib/python3.10/site-packages/yaml/composer.py:64: in compose_node
    if self.check_event(AliasEvent):
        index      = None
        parent     = None
        self       = <yaml.loader.SafeLoader object at 0x7f86ddf5b670>
/usr/lib/python3.10/site-packages/yaml/parser.py:98: in check_event
    self.current_event = self.state()
        choices    = (<class 'yaml.events.AliasEvent'>,)
        self       = <yaml.loader.SafeLoader object at 0x7f86ddf5b670>
/usr/lib/python3.10/site-packages/yaml/parser.py:211: in parse_document_content
    event = self.process_empty_scalar(self.peek_token().start_mark)
        self       = <yaml.loader.SafeLoader object at 0x7f86ddf5b670>
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <yaml.loader.SafeLoader object at 0x7f86ddf5b670>, mark = <yaml.error.Mark object at 0x7f86ddf5b520>

    def error_on_empty_scalar(self, mark):  # pylint: disable=unused-argument
        location = "{mark.name:s}:{mark.line:d} - column {mark.column:d}".format(mark=mark)
        error = "Error at {} - cannot define an empty value in test - either give it a value or explicitly set it to None".format(
            location
        )
    
>       raise exceptions.BadSchemaError(error)
E       tavern.util.exceptions.BadSchemaError: Error at <unicode string>:0 - column 3 - cannot define an empty value in test - either give it a value or explicitly set it to None

error      = ('Error at <unicode string>:0 - column 3 - cannot define an empty value in '
 'test - either give it a value or explicitly set it to None')
location   = '<unicode string>:0 - column 3'
mark       = <yaml.error.Mark object at 0x7f86ddf5b520>
self       = <yaml.loader.SafeLoader object at 0x7f86ddf5b670>

/usr/lib/python3.10/site-packages/tavern/util/loader.py:455: BadSchemaError

However, apispec never meant to use tavern, doesn’t specify anything that would request using tavern and I honestly doubt upstream would consider it a valid bug if I reported these test failures.

[mgorny]

This won’t work for Linux distribution packaging where all packages have to be installed globally.

[michaelboulton]

in 1.25.2, pyyaml will only be patched if a Tavern test is actually being loaded which should stop unwanted side effects.