How to change json encoding behaviour for serializable python object?

Question

It is easy to change the format of an object which is not JSON serializable eg datetime.datetime.

My requirement, for debugging purposes, is to alter the way some cu

Answer 1

The answer by FastTurtle might be a much cleaner solution.

Here's something close to what you want based on the technique as explained in my question/answer: Overriding nested JSON encoding of inherited default supported objects like dict, list

import json
import datetime


class mDict(dict):
    pass


class mList(list):
    pass


class JsonDebugEncoder(json.JSONEncoder):
    def _iterencode(self, o, markers=None):
        if isinstance(o, mDict):
            yield '{"__mDict__": '
            # Encode dictionary
            yield '{"orig": '
            for chunk in super(JsonDebugEncoder, self)._iterencode(o, markers):
                yield chunk
            yield ', '
            # / End of Encode dictionary
            # Encode attributes
            yield '"attr": '
            for key, value in o.__dict__.iteritems():
                yield '{"' + key + '": '
                for chunk in super(JsonDebugEncoder, self)._iterencode(value, markers):
                    yield chunk
                yield '}'
            yield '}'
            # / End of Encode attributes
            yield '}'
        elif isinstance(o, mList):
            yield '{"__mList__": '
            # Encode list
            yield '{"orig": '
            for chunk in super(JsonDebugEncoder, self)._iterencode(o, markers):
                yield chunk
            yield ', '
            # / End of Encode list
            # Encode attributes
            yield '"attr": '
            for key, value in o.__dict__.iteritems():
                yield '{"' + key + '": '
                for chunk in super(JsonDebugEncoder, self)._iterencode(value, markers):
                    yield chunk
                yield '}'
            yield '}'
            # / End of Encode attributes
            yield '}'
        else:
            for chunk in super(JsonDebugEncoder, self)._iterencode(o, markers=markers):
                yield chunk

    def default(self, obj):
        if isinstance(obj, datetime.datetime):
            return obj.isoformat()


class JsonDebugDecoder(json.JSONDecoder):
    def decode(self, s):
        obj = super(JsonDebugDecoder, self).decode(s)
        obj = self.recursiveObjectDecode(obj)
        return obj

    def recursiveObjectDecode(self, obj):
        if isinstance(obj, dict):
            decoders = [("__mList__", self.mListDecode),
                        ("__mDict__", self.mDictDecode)]
            for placeholder, decoder in decoders:
                if placeholder in obj:                  # We assume it's supposed to be converted
                    return decoder(obj[placeholder])
                else:
                    for k in obj:
                        obj[k] = self.recursiveObjectDecode(obj[k])
        elif isinstance(obj, list):
            for x in range(len(obj)):
                obj[x] = self.recursiveObjectDecode(obj[x])
        return obj

    def mDictDecode(self, o):
        res = mDict()
        for key, value in o['orig'].iteritems():
            res[key] = self.recursiveObjectDecode(value)
        for key, value in o['attr'].iteritems():
            res.__dict__[key] = self.recursiveObjectDecode(value)
        return res

    def mListDecode(self, o):
        res = mList()
        for value in o['orig']:
            res.append(self.recursiveObjectDecode(value))
        for key, value in o['attr'].iteritems():
            res.__dict__[key] = self.recursiveObjectDecode(value)
        return res


def test_debug_json():
    games = mList(['mario','contra','tetris'])
    games.src = 'console'
    scores = mDict({'dp':10,'pk':45})
    scores.processed = "unprocessed"
    test_json = { 'games' : games, 'scores' : scores ,'date': datetime.datetime.now() }
    jsonDump = json.dumps(test_json, cls=JsonDebugEncoder)
    print jsonDump
    test_pyObject = json.loads(jsonDump, cls=JsonDebugDecoder)
    print test_pyObject

if __name__ == '__main__':
    test_debug_json()

This results in:

{"date": "2013-05-06T22:28:08.967000", "games": {"__mList__": {"orig": ["mario", "contra", "tetris"], "attr": {"src": "console"}}}, "scores": {"__mDict__": {"orig": {"pk": 45, "dp": 10}, "attr": {"processed": "unprocessed"}}}}

This way you can encode it and decode it back to the python object it came from.

EDIT:

Here's a version that actually encodes it to the output you wanted and can decode it as well. Whenever a dictionary contains 'orig' and 'attr' it will check if 'orig' contains a dictionary or a list, if so it will respectively convert the object back to the mDict or mList.

import json
import datetime


class mDict(dict):
    pass


class mList(list):
    pass


class JsonDebugEncoder(json.JSONEncoder):
    def _iterencode(self, o, markers=None):
        if isinstance(o, mDict):    # Encode mDict
            yield '{"orig": '
            for chunk in super(JsonDebugEncoder, self)._iterencode(o, markers):
                yield chunk
            yield ', '
            yield '"attr": '
            for key, value in o.__dict__.iteritems():
                yield '{"' + key + '": '
                for chunk in super(JsonDebugEncoder, self)._iterencode(value, markers):
                    yield chunk
                yield '}'
            yield '}'
            # / End of Encode attributes
        elif isinstance(o, mList):    # Encode mList
            yield '{"orig": '
            for chunk in super(JsonDebugEncoder, self)._iterencode(o, markers):
                yield chunk
            yield ', '
            yield '"attr": '
            for key, value in o.__dict__.iteritems():
                yield '{"' + key + '": '
                for chunk in super(JsonDebugEncoder, self)._iterencode(value, markers):
                    yield chunk
                yield '}'
            yield '}'
        else:
            for chunk in super(JsonDebugEncoder, self)._iterencode(o, markers=markers):
                yield chunk

    def default(self, obj):
        if isinstance(obj, datetime.datetime):    # Encode datetime
            return obj.isoformat()


class JsonDebugDecoder(json.JSONDecoder):
    def decode(self, s):
        obj = super(JsonDebugDecoder, self).decode(s)
        obj = self.recursiveObjectDecode(obj)
        return obj

    def recursiveObjectDecode(self, obj):
        if isinstance(obj, dict):
            if "orig" in obj and "attr" in obj and isinstance(obj["orig"], list):
                return self.mListDecode(obj)
            elif "orig" in obj and "attr" in obj and isinstance(obj['orig'], dict):
                return self.mDictDecode(obj)
            else:
                for k in obj:
                    obj[k] = self.recursiveObjectDecode(obj[k])
        elif isinstance(obj, list):
            for x in range(len(obj)):
                obj[x] = self.recursiveObjectDecode(obj[x])
        return obj

    def mDictDecode(self, o):
        res = mDict()
        for key, value in o['orig'].iteritems():
            res[key] = self.recursiveObjectDecode(value)
        for key, value in o['attr'].iteritems():
            res.__dict__[key] = self.recursiveObjectDecode(value)
        return res

    def mListDecode(self, o):
        res = mList()
        for value in o['orig']:
            res.append(self.recursiveObjectDecode(value))
        for key, value in o['attr'].iteritems():
            res.__dict__[key] = self.recursiveObjectDecode(value)
        return res


def test_debug_json():
    games = mList(['mario','contra','tetris'])
    games.src = 'console'
    scores = mDict({'dp':10,'pk':45})
    scores.processed = "unprocessed"
    test_json = { 'games' : games, 'scores' : scores ,'date': datetime.datetime.now() }
    jsonDump = json.dumps(test_json, cls=JsonDebugEncoder)
    print jsonDump
    test_pyObject = json.loads(jsonDump, cls=JsonDebugDecoder)
    print test_pyObject
    print test_pyObject['games'].src

if __name__ == '__main__':
    test_debug_json()

Here's some more info about the output:

# Encoded
{"date": "2013-05-06T22:41:35.498000", "games": {"orig": ["mario", "contra", "tetris"], "attr": {"src": "console"}}, "scores": {"orig": {"pk": 45, "dp": 10}, "attr": {"processed": "unprocessed"}}}

# Decoded ('games' contains the mList with the src attribute and 'scores' contains the mDict processed attribute)
# Note that printing the python objects doesn't directly show the processed and src attributes, as seen below.
{u'date': u'2013-05-06T22:41:35.498000', u'games': [u'mario', u'contra', u'tetris'], u'scores': {u'pk': 45, u'dp': 10}}

Sorry for any bad naming conventions, it's a quick setup. ;)

Note: The datetime doesn't get decoded back to the python representation. Implementing that could be done by checking for any dict key that is called 'date' and contains a valid string representation of a datetime.