Problem with literal unroll of nested function tuples

hi, I’m on the latest master (c5c86c3a790e0b946ebbfdc3d94ff1d7214ec783), and the following code is failing.

import numpy as np
import numba
from numba import literal_unroll
@numba.njit
def foo1():
    return True

@numba.njit
def foo2():
    return False

@numba.njit
def bar1(x):
    return x

@numba.njit
def bar2(x):
    return x

x = (foo1, foo1, foo2)
y = (bar1, bar2, bar2)

funcs = tuple(zip(x, y))


@numba.njit
def baz(a, funcs):
    idx = 0
    for i in literal_unroll(funcs):
        j, k = i
        a[idx] = j() and k(2)
        idx += 1
    return x, a

baz(np.zeros((5, 10)), funcs)

The error is related to the unification of function types (I’m pasting the full trace at the bottom).

numba.core.errors.LoweringError: Failed in nopython mode pipeline (step: nopython mode backend)
unhashable type: 'list'

File "numba unification problem.py", line 30:
def baz(a, funcs):
    <source elided>
    idx = 0
    for i in literal_unroll(funcs):
    ^

[1] During: lowering "i.33 = typed_getitem(value=$8.3, dtype=Tuple(type(CPUDispatcher(<function foo1 at 0x7f11868ccea0>)), type(CPUDispatcher(<function bar1 at 0x7f11740aaae8>))), index=$phi18.1)" at /home/lucio/.PyCharm2018.3/config/scratches/numba unification problem.py (30)

As far as I can tell, the immediate problem is that utils.unified_function_type is returning a list, which gets passed to Unituple as dtype. This is not a valid dtype. Is a list ever a valid return from unified_function_type? If not, the error is in https://github.com/numba/numba/blob/0c976cd0ac640f2a06b756c7442f68abd2364e9c/numba/core/utils.py#L532, which returns the input argument (a list) unmodified. If returning the list is sometimes valid, then if t is not None: is wrong, since it should look into the list, not just pass it to Unituple.

You can see below what the input to unified_function_type is in this case.

cls = <class 'numba.core.types.containers.Tuple'>
types = [type(CPUDispatcher(<function foo1 at 0x1162699e0>)), UndefinedFunctionType[undefined(undefined, undefined, undefined, undefined, undefined)29]]

    def __new__(cls, types):

        t = utils.unified_function_type(types, require_precise=True)
        if t is not None:
>           return UniTuple(dtype=t, count=len(types))

../../numba/numba/numba/core/types/containers.py:282:

cc @pearu @stuartarchibald

This is the full error:

Traceback (most recent call last):
  File "/home/lucio/mypyprojects/numba/numba/core/errors.py", line 726, in new_error_context
    yield
  File "/home/lucio/mypyprojects/numba/numba/core/lowering.py", line 267, in lower_block
    self.lower_inst(inst)
  File "/home/lucio/mypyprojects/numba/numba/core/lowering.py", line 364, in lower_inst
    val = self.lower_assign(ty, inst)
  File "/home/lucio/mypyprojects/numba/numba/core/lowering.py", line 538, in lower_assign
    return self.lower_expr(ty, value)
  File "/home/lucio/mypyprojects/numba/numba/core/lowering.py", line 1194, in lower_expr
    self.loadvar(expr.index.name)))
  File "/home/lucio/mypyprojects/numba/numba/core/base.py", line 1155, in __call__
    res = self._imp(self._context, builder, self._sig, args, loc=loc)
  File "/home/lucio/mypyprojects/numba/numba/core/base.py", line 1185, in wrapper
    return fn(*args, **kwargs)
  File "/home/lucio/mypyprojects/numba/numba/cpython/tupleobj.py", line 254, in getitem_typed
    sig.return_type) == sig.return_type
  File "/home/lucio/mypyprojects/numba/numba/core/typing/context.py", line 630, in unify_types
    unified = self.unify_pairs(unified, tp)
  File "/home/lucio/mypyprojects/numba/numba/core/typing/context.py", line 649, in unify_pairs
    unified = first.unify(self, second)
  File "/home/lucio/mypyprojects/numba/numba/core/types/containers.py", line 320, in unify
    print(Tuple(unified))
  File "/home/lucio/mypyprojects/numba/numba/core/types/abstract.py", line 65, in __call__
    inst = type.__call__(cls, *args, **kwargs)
  File "/home/lucio/mypyprojects/numba/numba/core/types/containers.py", line 283, in __new__
    return UniTuple(dtype=t, count=len(types))
  File "/home/lucio/mypyprojects/numba/numba/core/types/abstract.py", line 66, in __call__
    return cls._intern(inst)
  File "/home/lucio/mypyprojects/numba/numba/core/types/abstract.py", line 50, in _intern
    orig = _typecache.get(wr)
  File "/home/lucio/mypyprojects/numba/numba/core/types/abstract.py", line 118, in __hash__
    return hash(self.key)
TypeError: unhashable type: 'list'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/lucio/.PyCharm2018.3/config/scratches/numba unification problem.py", line 37, in <module>
    baz(np.zeros((5, 10)), funcs)
  File "/home/lucio/mypyprojects/numba/numba/core/dispatcher.py", line 420, in _compile_for_args
    raise e
  File "/home/lucio/mypyprojects/numba/numba/core/dispatcher.py", line 353, in _compile_for_args
    return self.compile(tuple(argtypes))
  File "/home/lucio/mypyprojects/numba/numba/core/compiler_lock.py", line 32, in _acquire_compile_lock
    return func(*args, **kwargs)
  File "/home/lucio/mypyprojects/numba/numba/core/dispatcher.py", line 794, in compile
    cres = self._compiler.compile(args, return_type)
  File "/home/lucio/mypyprojects/numba/numba/core/dispatcher.py", line 77, in compile
    status, retval = self._compile_cached(args, return_type)
  File "/home/lucio/mypyprojects/numba/numba/core/dispatcher.py", line 91, in _compile_cached
    retval = self._compile_core(args, return_type)
  File "/home/lucio/mypyprojects/numba/numba/core/dispatcher.py", line 109, in _compile_core
    pipeline_class=self.pipeline_class)
  File "/home/lucio/mypyprojects/numba/numba/core/compiler.py", line 568, in compile_extra
    return pipeline.compile_extra(func)
  File "/home/lucio/mypyprojects/numba/numba/core/compiler.py", line 339, in compile_extra
    return self._compile_bytecode()
  File "/home/lucio/mypyprojects/numba/numba/core/compiler.py", line 401, in _compile_bytecode
    return self._compile_core()
  File "/home/lucio/mypyprojects/numba/numba/core/compiler.py", line 381, in _compile_core
    raise e
  File "/home/lucio/mypyprojects/numba/numba/core/compiler.py", line 372, in _compile_core
    pm.run(self.state)
  File "/home/lucio/mypyprojects/numba/numba/core/compiler_machinery.py", line 341, in run
    raise patched_exception
  File "/home/lucio/mypyprojects/numba/numba/core/compiler_machinery.py", line 332, in run
    self._runPass(idx, pass_inst, state)
  File "/home/lucio/mypyprojects/numba/numba/core/compiler_lock.py", line 32, in _acquire_compile_lock
    return func(*args, **kwargs)
  File "/home/lucio/mypyprojects/numba/numba/core/compiler_machinery.py", line 291, in _runPass
    mutated |= check(pss.run_pass, internal_state)
  File "/home/lucio/mypyprojects/numba/numba/core/compiler_machinery.py", line 264, in check
    mangled = func(compiler_state)
  File "/home/lucio/mypyprojects/numba/numba/core/typed_passes.py", line 442, in run_pass
    NativeLowering().run_pass(state)
  File "/home/lucio/mypyprojects/numba/numba/core/typed_passes.py", line 370, in run_pass
    lower.lower()
  File "/home/lucio/mypyprojects/numba/numba/core/lowering.py", line 179, in lower
    self.lower_normal_function(self.fndesc)
  File "/home/lucio/mypyprojects/numba/numba/core/lowering.py", line 227, in lower_normal_function
    entry_block_tail = self.lower_function_body()
  File "/home/lucio/mypyprojects/numba/numba/core/lowering.py", line 253, in lower_function_body
    self.lower_block(block)
  File "/home/lucio/mypyprojects/numba/numba/core/lowering.py", line 267, in lower_block
    self.lower_inst(inst)
  File "/home/lucio/anaconda3/envs/numbaenv/lib/python3.6/contextlib.py", line 99, in __exit__
    self.gen.throw(type, value, traceback)
  File "/home/lucio/mypyprojects/numba/numba/core/errors.py", line 733, in new_error_context
    reraise(type(newerr), newerr, tb)
  File "/home/lucio/mypyprojects/numba/numba/core/utils.py", line 81, in reraise
    raise value
numba.core.errors.LoweringError: Failed in nopython mode pipeline (step: nopython mode backend)
unhashable type: 'list'

File "numba unification problem.py", line 30:
def baz(a, funcs):
    <source elided>
    idx = 0
    for i in literal_unroll(funcs):
    ^

[1] During: lowering "i.33 = typed_getitem(value=$8.3, dtype=Tuple(type(CPUDispatcher(<function foo1 at 0x7f11868ccea0>)), type(CPUDispatcher(<function bar1 at 0x7f11740aaae8>))), index=$phi18.1)" at /home/lucio/.PyCharm2018.3/config/scratches/numba unification problem.py (30)

[luk-f-a]

thanks for sharing @pearu , that workaround looks very useful

cc @jrbourbeau

[pearu]

For whatever it’s worth, a workaround to this bug is:

funcs = tuple(zip(range(len(x)), x, y))

@numba.njit
def baz(a, funcs):
    for i in literal_unroll(funcs):
        idx, j, k = i
        a[idx] = j() and k(2)
    return a

baz(np.zeros((5, 10)), funcs)