python exceptions are not json serializable.

I have some celery tasks, which may raise exception, if required. The problem is python Exception object is not json-serialization, yet though yaml can serialize python Exception objects but I don’t have control to change this configuration for celery.

Can we in celery(more specifically in kombu) internally handle, serialization of exception object ?

Here is a snippet which will generate error.

from kombu import Connection

ex = None
try:
    a = 1/0
except ZeroDivisionError, e:
    ex = e
print ex

conn = Connection('amqp://guest:guest@localhost:5672//')
simple_queue = conn.SimpleQueue('simple_queue')

# No error as expected
simple_queue.put('Hello')

# No error as expected
simple_queue.put(str(ex))

# json cannot serliaze exception
# hence celery gives error
simple_queue.put(ex)

Here is error which I get.

---------------------------------------------------------------------------
EncodeError                               Traceback (most recent call last)
<ipython-input-22-9667148d81ba> in <module>()
----> 1 sq.put(ex)

/usr/local/lib/python2.7/dist-packages/kombu/simple.pyc in put(self, message, serializer, headers, compression, routing_key, **kwargs)
     70                               headers=headers,
     71                               compression=compression,
---> 72                               **kwargs)
     73 
     74     def clear(self):

/usr/local/lib/python2.7/dist-packages/kombu/messaging.pyc in publish(self, body, routing_key, delivery_mode, mandatory, immediate, priority, content_type, content_encoding, serializer, headers, compression, exchange, retry, retry_policy, declare, expiration, **properties)
    163         body, content_type, content_encoding = self._prepare(
    164             body, serializer, content_type, content_encoding,
--> 165             compression, headers)
    166 
    167         publish = self._publish

/usr/local/lib/python2.7/dist-packages/kombu/messaging.pyc in _prepare(self, body, serializer, content_type, content_encoding, compression, headers)
    239             serializer = serializer or self.serializer
    240             (content_type, content_encoding,
--> 241              body) = dumps(body, serializer=serializer)
    242         else:
    243             # If the programmer doesn't want us to serialize,

/usr/local/lib/python2.7/dist-packages/kombu/serialization.pyc in dumps(self, data, serializer)
    162 
    163         with _reraise_errors(EncodeError):
--> 164             payload = encoder(data)
    165         return content_type, content_encoding, payload
    166     encode = dumps  # XXX compat

/usr/lib/python2.7/contextlib.pyc in __exit__(self, type, value, traceback)
     33                 value = type()
     34             try:
---> 35                 self.gen.throw(type, value, traceback)
     36                 raise RuntimeError("generator didn't stop after throw()")
     37             except StopIteration, exc:

/usr/local/lib/python2.7/dist-packages/kombu/serialization.pyc in _reraise_errors(wrapper, include, exclude)
     57         raise
     58     except include as exc:
---> 59         reraise(wrapper, wrapper(exc), sys.exc_info()[2])
     60 
     61 

/usr/local/lib/python2.7/dist-packages/kombu/serialization.pyc in _reraise_errors(wrapper, include, exclude)
     53                     include=(Exception, ), exclude=(SerializerNotInstalled, )):
     54     try:
---> 55         yield
     56     except exclude:
     57         raise

/usr/local/lib/python2.7/dist-packages/kombu/serialization.pyc in dumps(self, data, serializer)
    162 
    163         with _reraise_errors(EncodeError):
--> 164             payload = encoder(data)
    165         return content_type, content_encoding, payload
    166     encode = dumps  # XXX compat

/usr/local/lib/python2.7/dist-packages/anyjson/__init__.pyc in dumps(value)
    139     def dumps(value):
    140         """Deserialize JSON-encoded object to a Python object."""
--> 141         return implementation.dumps(value)
    142     serialize = dumps

/usr/local/lib/python2.7/dist-packages/anyjson/__init__.pyc in dumps(self, data)
     85         TypeError if the object could not be serialized."""
     86         try:
---> 87             return self._encode(data)
     88         except self._encode_error, exc:
     89             raise TypeError, TypeError(*exc.args), sys.exc_info()[2]

/usr/local/lib/python2.7/dist-packages/simplejson-3.8.1-py2.7-linux-x86_64.egg/simplejson/__init__.pyc in dumps(obj, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, encoding, default, use_decimal, namedtuple_as_object, tuple_as_array, bigint_as_string, sort_keys, item_sort_key, for_json, ignore_nan, int_as_string_bitcount, iterable_as_array, **kw)
    378         and not kw
    379     ):
--> 380         return _default_encoder.encode(obj)
    381     if cls is None:
    382         cls = JSONEncoder

/usr/local/lib/python2.7/dist-packages/simplejson-3.8.1-py2.7-linux-x86_64.egg/simplejson/encoder.pyc in encode(self, o)
    273         # exceptions aren't as detailed.  The list call should be roughly
    274         # equivalent to the PySequence_Fast that ''.join() would do.
--> 275         chunks = self.iterencode(o, _one_shot=True)
    276         if not isinstance(chunks, (list, tuple)):
    277             chunks = list(chunks)

/usr/local/lib/python2.7/dist-packages/simplejson-3.8.1-py2.7-linux-x86_64.egg/simplejson/encoder.pyc in iterencode(self, o, _one_shot)
    355                 self.iterable_as_array, Decimal=decimal.Decimal)
    356         try:
--> 357             return _iterencode(o, 0)
    358         finally:
    359             key_memo.clear()

/usr/local/lib/python2.7/dist-packages/simplejson-3.8.1-py2.7-linux-x86_64.egg/simplejson/encoder.pyc in default(self, o)
    250 
    251         """
--> 252         raise TypeError(repr(o) + " is not JSON serializable")
    253 
    254     def encode(self, o):

[mac10046]

Hey guys I guess this should work good enough for any Exception which is not serializable

    except Exception as e:
        import json
        json.dumps(e.__dict__)

This worked for me atleast.

[georgepsarakis]

@humitos you can use self.backend.store_result, passing the exception instance, as seen here. This is a different use case, so please open a separate issue, so that we have better tracking and knowledge base. Let me know if this works!