Documenting CuPy wrapper progress

Starting this issue to document progress on wrapping CuPy.

• import autograd.cupy as cp
• instantiate arrays from scalars, lists, and tuples.

cp.array(1)
cp.array([1, 2])
cp.array([1, 3]) + cp.array([1, 1])

• check that gradients work

import autograd.cupy as cp
from autograd import elementwise_grad as egrad

def f(x):
    return cp.sin(x)

def g(x):
    return x + 2

df = egrad(f)
dg = egrad(g)

a = cp.array([1, 1])

print(f(a))
print(df(a))

print(g(a))
print(dg(a))

• Check that higher derivatives work.

import autograd.cupy as cp
from autograd import elementwise_grad as egrad
import numpy as np

a = cp.arange(-2 * np.pi, 2 * np.pi, 0.01)

def sin(x):
    return cp.sin(x)

dsin = egrad(sin)
ddsin = egrad(dsin)

sin(a)
dsin(a)
ddsin(a)

• Fix ValueError: object __array__ method not producing an array.
• Run tests for all of the CuPy wrapped functions.

[ericmjl]

Running the tests for CuPy, there’s at least some progress!

tests/test_cupy.py ...FFFFFFFFFF...FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF....FF.FFFFFFFF.FF.FF..FFFFFFFFFFFFFFFFFFFFFF.FFFFFF.F                                                                               [100%]

[ericmjl]

Here’s the latest example:

import autograd.numpy as np
import autograd.cupy as cp
from autograd import elementwise_grad as egrad

a = cp.arange(-2.0 * np.pi, 2.0 * np.pi, 0.01)

a = cp.arange(-2.0 * np.pi, 2.0 * np.pi, 0.01)

def f(x):
    return cp.sin(x)

df = egrad(f)
ddf = egrad(df)

ddf(a)
# Output: array([-2.44929360e-16, -9.99983333e-03, -1.99986667e-02, ...,
        2.63675581e-02,  1.63698832e-02,  6.37057127e-03])

I have encountered other problems though, an example:

df = egrad(f)
df(cp.array([1, 2, 3, 4, 5]))
# Output: array([4.19974342e-01, 7.06508249e-02, 9.86603717e-03, 1.34095068e-03,
       1.81583231e-04])

ddf = egrad(df)

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-27-ba44d2096557> in <module>()
      2 df(a)
      3 ddf = egrad(df)
----> 4 ddf(a)
      5 

~/github/software/autograd/autograd/wrap_util.py in nary_f(*args, **kwargs)
     18             else:
     19                 x = tuple(args[i] for i in argnum)
---> 20             return unary_operator(unary_f, x, *nary_op_args, **nary_op_kwargs)
     21         return nary_f
     22     return nary_operator

~/github/software/autograd/autograd/differential_operators.py in elementwise_grad(fun, x)
     34     if vspace(ans).iscomplex:
     35         raise TypeError("Elementwise_grad only applies to real-output functions.")
---> 36     return vjp(vspace(ans).ones())
     37 
     38 @unary_to_nary

~/github/software/autograd/autograd/core.py in vjp(g)
     12         def vjp(g): return vspace(x).zeros()
     13     else:
---> 14         def vjp(g): return backward_pass(g, end_node)
     15     return vjp, end_value
     16 

~/github/software/autograd/autograd/core.py in backward_pass(g, end_node)
     19     for node in toposort(end_node):
     20         outgrad = outgrads.pop(node)
---> 21         ingrads = node.vjp(outgrad[0])
     22         for parent, ingrad in zip(node.parents, ingrads):
     23             outgrads[parent] = add_outgrads(outgrads.get(parent), ingrad)

~/github/software/autograd/autograd/core.py in <lambda>(g)
     65                     "VJP of {} wrt argnum 0 not defined".format(fun.__name__))
     66             vjp = vjpfun(ans, *args, **kwargs)
---> 67             return lambda g: (vjp(g),)
     68         elif L == 2:
     69             argnum_0, argnum_1 = argnums

~/github/software/autograd/autograd/cupy/cupy_vjps.py in <lambda>(g)
   1028 def unbroadcast_f(target, f):
   1029     target_meta = acp.metadata(target)
-> 1030     return lambda g: unbroadcast(f(g), target_meta)
   1031 
   1032 

~/github/software/autograd/autograd/cupy/cupy_vjps.py in <lambda>(g)
    195     lambda ans,
    196     x,
--> 197     y: unbroadcast_f(x, lambda g: g * y * x ** acp.where(y, y - 1., 1.)),
    198     lambda ans,
    199     x,

~/github/software/autograd/autograd/tracer.py in f_wrapped(*args, **kwargs)
     46             return new_box(ans, trace, node)
     47         else:
---> 48             return f_raw(*args, **kwargs)
     49     f_wrapped.fun = f_raw
     50     f_wrapped._is_autograd_primitive = True

~/anaconda/envs/autograd_cupy_dev/lib/python3.6/site-packages/cupy/core/fusion.py in __call__(self, *args, **kwargs)
    699                 return self._numpy_op(*args, **kwargs)
    700 
--> 701         return self._cupy_op(*args, **kwargs)
    702 
    703     __doc__ = core.ufunc.__doc__

~/anaconda/envs/autograd_cupy_dev/lib/python3.6/site-packages/cupy/sorting/search.py in where(condition, x, y)
    119         return nonzero(condition)
    120 
--> 121     return _where_ufunc(condition.astype('?'), x, y)
    122 
    123 

AttributeError: 'int' object has no attribute 'astype'