BUG in read_pickle

Summary: read_pickle from Pandas version > 1.25 is unable to load pickles created with pandas version 0.24 and Python 2.7. The documentation claims it is compatible with Pandas versions starting in 0.20, and it does work with pandas version until 1.25 and Python 3.9

The desired behaviour is that it should work the same it did until version 1.2.5, as is it still described in the documentation.

Example:

in Python 2.7 with Pandas 0.24 run:

import cPickle as pickle
import pandas as pd

d = pd.DataFrame()

with open("test_pickle.pkl", 'wb') as f:
    pickle.dump(d, f)

Now, in Python 3.9, with Pandas 1.25 run:

import pandas as pd

with open("test_pickle.pkl", 'rb') as f:
    d=pd.read_pickle(f)

print(d)

You get the correct output:

Empty DataFrame
Columns: []
Index: []

Now repeat in Python 3.9, Pandas 1.3:

import pandas as pd

with open("test_pickle.pkl", 'rb') as f:
    d=pd.read_pickle(f)

You get the error:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
File ~/miniconda3/envs/test6/lib/python3.9/site-packages/pandas/io/pickle.py:217, in read_pickle(filepath_or_buffer, compression, storage_options)
    214         # error: Argument 1 to "load" has incompatible type "Union[IO[Any],
    215         # RawIOBase, BufferedIOBase, TextIOBase, TextIOWrapper, mmap]";
    216         # expected "IO[bytes]"
--> 217         return pickle.load(handles.handle)  # type: ignore[arg-type]
    218 except excs_to_catch:
    219     # e.g.
    220     #  "No module named 'pandas.core.sparse.series'"
    221     #  "Can't get attribute '__nat_unpickle' on <module 'pandas._libs.tslib"

File ~/miniconda3/envs/test6/lib/python3.9/copyreg.py:43, in _reconstructor(cls, base, state)
     42 if base is object:
---> 43     obj = object.__new__(cls)
     44 else:

TypeError: object.__new__(BlockManager) is not safe, use BlockManager.__new__()

During handling of the above exception, another exception occurred:

TypeError                                 Traceback (most recent call last)
Input In [2], in <cell line: 1>()
      1 with open("test_pickle.pkl", 'rb') as f:
----> 2     d=pd.read_pickle(f)

File ~/miniconda3/envs/test6/lib/python3.9/site-packages/pandas/io/pickle.py:222, in read_pickle(filepath_or_buffer, compression, storage_options)
    217             return pickle.load(handles.handle)  # type: ignore[arg-type]
    218     except excs_to_catch:
    219         # e.g.
    220         #  "No module named 'pandas.core.sparse.series'"
    221         #  "Can't get attribute '__nat_unpickle' on <module 'pandas._libs.tslib"
--> 222         return pc.load(handles.handle, encoding=None)
    223 except UnicodeDecodeError:
    224     # e.g. can occur for files written in py27; see GH#28645 and GH#31988
    225     return pc.load(handles.handle, encoding="latin-1")

File ~/miniconda3/envs/test6/lib/python3.9/site-packages/pandas/compat/pickle_compat.py:274, in load(fh, encoding, is_verbose)
    271         up = Unpickler(fh)
    272     up.is_verbose = is_verbose
--> 274     return up.load()
    275 except (ValueError, TypeError):
    276     raise

File ~/miniconda3/envs/test6/lib/python3.9/pickle.py:1212, in _Unpickler.load(self)
   1210             raise EOFError
   1211         assert isinstance(key, bytes_types)
-> 1212         dispatch[key[0]](self)
   1213 except _Stop as stopinst:
   1214     return stopinst.value

File ~/miniconda3/envs/test6/lib/python3.9/site-packages/pandas/compat/pickle_compat.py:42, in load_reduce(self)
     39     n = args[0].__name__  # noqa
     41 try:
---> 42     stack[-1] = func(*args)
     43     return
     44 except TypeError as err:
     45 
     46     # If we have a deprecated function,
     47     # try to replace and try again.

File ~/miniconda3/envs/test6/lib/python3.9/copyreg.py:43, in _reconstructor(cls, base, state)
     41 def _reconstructor(cls, base, state):
     42     if base is object:
---> 43         obj = object.__new__(cls)
     44     else:
     45         obj = base.__new__(cls, state)

TypeError: object.__new__(BlockManager) is not safe, use BlockManager.__new__()

[MarcoGorelli]

Sure, go ahead - if you comment “take” it’ll be assigned to you

[MarcoGorelli]

Does it work if you save it with to_pickle? I think that’s what the docs note assumes you used to create the pickle