Stuck on an issue?

Lightrun Answers was designed to reduce the constant googling that comes with debugging 3rd party libraries. It collects links to all the places you might be looking at while hunting down a tough bug.

And, if you’re still stuck at the end, we’re happy to hop on a call to see how we can help out.

Using progressbar raise an error: OverflowError: int too big to convert

See original GitHub issue

OverflowError                             Traceback (most recent call last)
<timed exec> in <module>

~/.conda/envs/work/lib/python3.9/site-packages/pandarallel/pandarallel.py in closure(data, func, *args, **kwargs)
    429         queue = manager.Queue()
    430 
--> 431         workers_args, chunk_lengths, input_files, output_files = get_workers_args(
    432             use_memory_fs,
    433             nb_requested_workers,

~/.conda/envs/work/lib/python3.9/site-packages/pandarallel/pandarallel.py in get_workers_args(use_memory_fs, nb_workers, progress_bar, chunks, worker_meta_args, queue, func, args, kwargs)
    284             raise OSError(msg)
    285 
--> 286         workers_args = [
    287             (
    288                 input_file.name,

~/.conda/envs/work/lib/python3.9/site-packages/pandarallel/pandarallel.py in <listcomp>(.0)
    293                 progress_bar == PROGRESS_IN_WORKER,
    294                 dill.dumps(
--> 295                     progress_wrapper(
    296                         progress_bar >= PROGRESS_IN_FUNC, queue, index, chunk_length
    297                     )(func)

~/.conda/envs/work/lib/python3.9/site-packages/pandarallel/pandarallel.py in wrapper(func)
    203     def wrapper(func):
    204         if progress_bar:
--> 205             wrapped_func = inline(
    206                 progress_pre_func,
    207                 func,

~/.conda/envs/work/lib/python3.9/site-packages/pandarallel/utils/inliner.py in wrapper(*args, **kwargs)
     32             raise SystemError("Python version should be 3.{5, 6, 7, 8, 9}")
     33 
---> 34         return function(*args, **kwargs)
     35 
     36     return wrapper

~/.conda/envs/work/lib/python3.9/site-packages/pandarallel/utils/inliner.py in inline(pre_func, func, pre_func_arguments)
    485 
    486     func_instructions = tuple(get_instructions(func))
--> 487     shifted_func_instructions = shift_instructions(
    488         func_instructions, len(b"".join(pinned_pre_func_instructions_without_return))
    489     )

~/.conda/envs/work/lib/python3.9/site-packages/pandarallel/utils/inliner.py in wrapper(*args, **kwargs)
     32             raise SystemError("Python version should be 3.{5, 6, 7, 8, 9}")
     33 
---> 34         return function(*args, **kwargs)
     35 
     36     return wrapper

~/.conda/envs/work/lib/python3.9/site-packages/pandarallel/utils/inliner.py in shift_instructions(instructions, qty)
    301     If Python version not in 3.{5, 6, 7}, a SystemError is raised.
    302     """
--> 303     return tuple(
    304         shift_instruction(instruction, qty)
    305         if bytes((instruction[0],))

~/.conda/envs/work/lib/python3.9/site-packages/pandarallel/utils/inliner.py in <genexpr>(.0)
    302     """
    303     return tuple(
--> 304         shift_instruction(instruction, qty)
    305         if bytes((instruction[0],))
    306         in (

~/.conda/envs/work/lib/python3.9/site-packages/pandarallel/utils/inliner.py in wrapper(*args, **kwargs)
     32             raise SystemError("Python version should be 3.{5, 6, 7, 8, 9}")
     33 
---> 34         return function(*args, **kwargs)
     35 
     36     return wrapper

~/.conda/envs/work/lib/python3.9/site-packages/pandarallel/utils/inliner.py in shift_instruction(instruction, qty)
    291     """
    292     operation, *values = instruction
--> 293     return bytes((operation,)) + int2python_bytes(python_ints2int(values) + qty)
    294 
    295 

~/.conda/envs/work/lib/python3.9/site-packages/pandarallel/utils/inliner.py in wrapper(*args, **kwargs)
     32             raise SystemError("Python version should be 3.{5, 6, 7, 8, 9}")
     33 
---> 34         return function(*args, **kwargs)
     35 
     36     return wrapper

~/.conda/envs/work/lib/python3.9/site-packages/pandarallel/utils/inliner.py in int2python_bytes(item)
     69 
     70     nb_bytes = 2 if python_version.minor == 5 else 1
---> 71     return int.to_bytes(item, nb_bytes, "little")
     72 
     73 

OverflowError: int too big to convert