[PATCH] re-applying the patch that really matters to me in hope that it will be easier to merge back into origin
Dan Buch
src/benchmark.py | 22 ++++++++----
src/jsonpickle/__init__.py | 14 ++++---
src/jsonpickle/pickler.py | 8 +++-
src/jsonpickle/tests/__init__.py | 4 +-
src/jsonpickle/tests/classes.py | 50 +++++++++++++++++++++++++++--
src/jsonpickle/tests/jsonpickle_test.py | 21 +++++++++++-
src/jsonpickle/tests/thirdparty_tests.py | 2 +-
src/jsonpickle/tests/util_tests.py | 28 ++++++++--------
src/jsonpickle/unpickler.py | 32 ++++++++++--------
src/setup.py | 52 ++++++++++++++++++++----------
10 files changed, 165 insertions(+), 68 deletions(-)
diff --git a/src/benchmark.py b/src/benchmark.py
index 866f287..f3a9676 100755
--- a/src/benchmark.py
+++ b/src/benchmark.py
@@ -7,8 +7,11 @@
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution.
+import sys
import timeit
+IS_25_DOWN = sys.version_info[:2] <= (2, 5)
+
number = 1000
cjson = """\
@@ -17,7 +20,7 @@ import jsonpickle
import jsonpickle.tests.thirdparty_tests as test
doc = feedparser.parse(test.RSS_DOC)
-jsonpickle._use_cjson()
+jsonpickle.set_preferred_backend('cjson')
pickled = jsonpickle.encode(doc)
unpickled = jsonpickle.decode(pickled)
@@ -30,21 +33,24 @@ cjson_test = timeit.Timer(stmt=cjson)
print "%.9f sec/pass " % (cjson_test.timeit(number=number) / number)
-simplejson = """\
+mod = 'json'
+if IS_25_DOWN:
+ mod = 'simplejson'
+
+json = """\
import feedparser
import jsonpickle
import jsonpickle.tests.thirdparty_tests as test
doc = feedparser.parse(test.RSS_DOC)
-jsonpickle._use_simplejson()
+jsonpickle.set_preferred_backend('%s')
pickled = jsonpickle.encode(doc)
unpickled = jsonpickle.decode(pickled)
if doc['feed']['title'] != unpickled['feed']['title']:
print 'Not a match'
-"""
-
-print 'Using simplejson'
-simplejson_test = timeit.Timer(stmt=simplejson)
-print "%.9f sec/pass " % (simplejson_test.timeit(number=number) / number)
+""" % mod
+print 'Using %s' % mod
+json_test = timeit.Timer(stmt=json)
+print "%.9f sec/pass " % (json_test.timeit(number=number) / number)
diff --git a/src/jsonpickle/__init__.py b/src/jsonpickle/__init__.py
index 2fc6574..d563eee 100644
--- a/src/jsonpickle/__init__.py
+++ b/src/jsonpickle/__init__.py
@@ -178,7 +178,7 @@ class JSONPluginMgr(object):
self._backend_names.remove(name)
self._verified = bool(self._backend_names)
- def encode(self, obj):
+ def encode(self, obj, **kwargs):
"""Attempts to encode an object into JSON.
This tries the loaded backends in order and passes along the last
@@ -187,10 +187,12 @@ class JSONPluginMgr(object):
self._verify()
for idx, name in enumerate(self._backend_names):
try:
- optargs, kwargs = self._encoder_options[name]
- args = (obj,) + tuple(optargs)
- return self._encoders[name](*args, **kwargs)
- except:
+ optargs, optkwargs = self._encoder_options[name]
+ encoder_kwargs = optkwargs.copy()
+ encoder_kwargs.update(kwargs)
+ encoder_args = (obj,) + tuple(optargs)
+ return self._encoders[name](*encoder_args, **encoder_kwargs)
+ except Exception:
if idx == len(self._backend_names) - 1:
raise
@@ -287,7 +289,7 @@ def encode(value, unpicklable=True, max_depth=None, **kwargs):
"""
j = Pickler(unpicklable=unpicklable,
max_depth=max_depth)
- return json.encode(j.flatten(value))
+ return json.encode(j.flatten(value), **kwargs)
def decode(string):
"""Converts the JSON string into a Python object.
diff --git a/src/jsonpickle/pickler.py b/src/jsonpickle/pickler.py
index a8e1a78..0ee8807 100644
--- a/src/jsonpickle/pickler.py
+++ b/src/jsonpickle/pickler.py
@@ -125,6 +125,7 @@ class Pickler(object):
data = {}
has_class = hasattr(obj, '__class__')
has_dict = hasattr(obj, '__dict__')
+ has_getstate = hasattr(obj, '__getstate__')
if self._mkref(obj):
if (has_class and not util.is_repr(obj) and
not util.is_module(obj)):
@@ -154,11 +155,14 @@ class Pickler(object):
if util.is_noncomplex(obj):
return self._pop([self.flatten(v) for v in obj])
- if has_dict:
+ if has_dict or has_getstate:
if util.is_collection_subclass(obj):
self._flatten_collection_obj(obj, data)
return self._pop(data)
- return self._pop(self._flatten_dict_obj(obj.__dict__, data))
+ obj_dict = obj.__dict__
+ if has_getstate:
+ obj_dict = obj.__getstate__()
+ return self._pop(self._flatten_dict_obj(obj_dict, data))
else:
# We've seen this object before so place an object
# reference tag in the data. This avoids infinite recursion
diff --git a/src/jsonpickle/tests/__init__.py b/src/jsonpickle/tests/__init__.py
index 4a98114..528d0f1 100755
--- a/src/jsonpickle/tests/__init__.py
+++ b/src/jsonpickle/tests/__init__.py
@@ -13,7 +13,7 @@ import jsonpickle.tests.jsonpickle_test
import jsonpickle.tests.thirdparty_tests
def suite():
- suite = unittest.TestSuite()
+ suite = unittest.TestSuite()
suite.addTest(jsonpickle.tests.util_tests.suite())
suite.addTest(jsonpickle.tests.jsonpickle_test.suite())
suite.addTest(jsonpickle.tests.thirdparty_tests.suite())
@@ -22,6 +22,6 @@ def suite():
def main():
#unittest.main(defaultTest='suite')
unittest.TextTestRunner(verbosity=2).run(suite())
-
+
if __name__ == '__main__':
main()
\ No newline at end of file
diff --git a/src/jsonpickle/tests/classes.py b/src/jsonpickle/tests/classes.py
index 150e25d..0bfee19 100644
--- a/src/jsonpickle/tests/classes.py
+++ b/src/jsonpickle/tests/classes.py
@@ -10,13 +10,57 @@ class Thing(object):
def __init__(self, name):
self.name = name
self.child = None
-
+
def __repr__(self):
return 'jsonpickle.tests.classes.Thing("%s")' % self.name
-
+
+
+class HasProps(object):
+
+ def __init__(self, name='', dogs='reliable', monkies='tricksy'):
+ self.name = name
+ self._critters = (('dogs', dogs), ('monkies', monkies))
+
+ def _get_identity(self):
+ keys = [self.dogs, self.monkies, self.name]
+ return hash('-'.join([str(key) for key in keys]))
+
+ identity = property(_get_identity)
+
+ def _get_dogs(self):
+ return self._critters[0][1]
+
+ dogs = property(_get_dogs)
+
+ def _get_monkies(self):
+ return self._critters[1][1]
+
+ monkies = property(_get_monkies)
+
+ def __getstate__(self):
+ out = dict(
+ __identity__=self.identity,
+ nom=self.name,
+ dogs=self.dogs,
+ monkies=self.monkies,
+ )
+ return out
+
+ def __setstate__(self, state_dict):
+ self._critters = (('dogs', state_dict.get('dogs')),
+ ('monkies', state_dict.get('monkies')))
+ self.name = state_dict.get('nom', '')
+ ident = state_dict.get('__identity__')
+ if ident != self.identity:
+ raise ValueError('expanded object does not match originial state!')
+
+ def __eq__(self, other):
+ return self.identity == other.identity
+
+
class DictSubclass(dict):
name = 'Test'
-
+
class ListSubclass(list):
pass
diff --git a/src/jsonpickle/tests/jsonpickle_test.py b/src/jsonpickle/tests/jsonpickle_test.py
index b2e60c7..84a7a58 100644
--- a/src/jsonpickle/tests/jsonpickle_test.py
+++ b/src/jsonpickle/tests/jsonpickle_test.py
@@ -7,6 +7,7 @@
# you should have received as part of this distribution.
import os
+import sys
import doctest
import unittest
import datetime
@@ -15,7 +16,7 @@ import time
import jsonpickle
from jsonpickle import tags
-from jsonpickle.tests.classes import Thing
+from jsonpickle.tests.classes import Thing, HasProps
from jsonpickle.tests.classes import BrokenReprThing
from jsonpickle.tests.classes import DictSubclass
from jsonpickle.tests.classes import ListSubclass
@@ -384,6 +385,14 @@ class PicklingTestCase(unittest.TestCase):
inflated = self.unpickler.restore(flattened)
self.assertEqual(inflated.classref, Thing)
+ def test_supports_getstate_setstate(self):
+ obj = HasProps('object-which-defines-getstate-setstate')
+ flattened = self.pickler.flatten(obj)
+ self.assertTrue(flattened.get('__identity__'))
+ self.assertTrue(flattened.get('nom'))
+ inflated = self.unpickler.restore(flattened)
+ self.assertEqual(obj, inflated)
+
class JSONPickleTestCase(unittest.TestCase):
def setUp(self):
@@ -399,6 +408,16 @@ class JSONPickleTestCase(unittest.TestCase):
pickled = jsonpickle.encode(self.obj, unpicklable=False)
self.assertEqual('{"name": "A name", "child": null}', pickled)
+ def test_encode_passes_through_kwargs(self):
+ preferred_backend = 'simplejson'
+ if sys.version_info[:2] >= (2, 6):
+ preferred_backend = 'json'
+ jsonpickle.set_preferred_backend(preferred_backend)
+ pickled = jsonpickle.encode(self.obj, indent=4)
+ self.assertEqual('{\n "py/object": "jsonpickle.tests.classes.Thing"'
+ ', \n "name": "A name", \n "child": null\n}',
+ pickled)
+
def test_decode(self):
unpickled = jsonpickle.decode(self.expected_json)
self.assertEqual(self.obj.name, unpickled.name)
diff --git a/src/jsonpickle/tests/thirdparty_tests.py b/src/jsonpickle/tests/thirdparty_tests.py
index ae03f35..be75232 100644
--- a/src/jsonpickle/tests/thirdparty_tests.py
+++ b/src/jsonpickle/tests/thirdparty_tests.py
@@ -61,7 +61,7 @@ class FeedParserTest(unittest.TestCase):
def setUp(self):
#self.doc = feedparser.parse("http://feedparser.org/docs/examples/atom10.xml")
self.doc = feedparser.parse(RSS_DOC)
-
+
def test(self):
pickled = jsonpickle.encode(self.doc)
unpickled = jsonpickle.decode(pickled)
diff --git a/src/jsonpickle/tests/util_tests.py b/src/jsonpickle/tests/util_tests.py
index 1e223ef..8e7987b 100644
--- a/src/jsonpickle/tests/util_tests.py
+++ b/src/jsonpickle/tests/util_tests.py
@@ -67,18 +67,18 @@ class IsPrimitiveTestCase(unittest.TestCase):
class IsCollection(unittest.TestCase):
def test_list(self):
self.assertTrue(is_list([1, 2]))
-
+
def test_set(self):
self.assertTrue(is_set(set([1, 2])))
-
+
def test_tuple(self):
self.assertTrue(is_tuple((1, 2)))
-
+
def test_dict(self):
self.assertFalse(is_list({'key':'value'}))
self.assertFalse(is_set({'key':'value'}))
self.assertFalse(is_tuple({'key':'value'}))
-
+
def test_other(self):
self.assertFalse(is_list(1))
self.assertFalse(is_set(1))
@@ -87,28 +87,28 @@ class IsCollection(unittest.TestCase):
class IsDictionary(unittest.TestCase):
def test_dict(self):
self.assertTrue(is_dictionary({'key':'value'}))
-
+
def test_list(self):
self.assertFalse(is_dictionary([1, 2]))
class IsDictionarySubclass(unittest.TestCase):
def test_subclass(self):
self.assertTrue(is_dictionary_subclass(DictSubclass()))
-
+
def test_dict(self):
self.assertFalse(is_dictionary_subclass({'key':'value'}))
class IsCollectionSubclass(unittest.TestCase):
def test_subclass(self):
self.assertTrue(is_collection_subclass(ListSubclass()))
-
+
def test_list(self):
self.assertFalse(is_collection_subclass([]))
class IsNonComplex(unittest.TestCase):
def setUp(self):
self.time = time.struct_time('123456789')
-
+
def test_time_struct(self):
self.assertTrue(is_noncomplex(self.time))
@@ -118,22 +118,22 @@ class IsNonComplex(unittest.TestCase):
class IsRepr(unittest.TestCase):
def setUp(self):
self.time = datetime.datetime.now()
-
+
def test_datetime(self):
self.assertTrue(is_repr(self.time))
-
+
def test_date(self):
self.assertTrue(is_repr(self.time.date()))
-
+
def test_time(self):
self.assertTrue(is_repr(self.time.time()))
-
+
def test_timedelta(self):
self.assertTrue(is_repr(datetime.timedelta(4)))
-
+
def test_object(self):
self.assertFalse(is_repr(object()))
-
+
def suite():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(IsPrimitiveTestCase))
diff --git a/src/jsonpickle/unpickler.py b/src/jsonpickle/unpickler.py
index ae96685..edd3b72 100644
--- a/src/jsonpickle/unpickler.py
+++ b/src/jsonpickle/unpickler.py
@@ -84,20 +84,24 @@ class Unpickler(object):
# keep a obj->name mapping for use in the _isobjref() case
self._mkref(instance)
- for k, v in obj.iteritems():
- # ignore the reserved attribute
- if k in tags.RESERVED:
- continue
- self._namestack.append(k)
- # step into the namespace
- value = self.restore(v)
- if (util.is_noncomplex(instance) or
- util.is_dictionary_subclass(instance)):
- instance[k] = value
- else:
- instance.__dict__[k] = value
- # step out
- self._namestack.pop()
+ if hasattr(instance, '__setstate__'):
+ instance.__setstate__(obj)
+
+ else:
+ for k, v in obj.iteritems():
+ # ignore the reserved attribute
+ if k in tags.RESERVED:
+ continue
+ self._namestack.append(k)
+ # step into the namespace
+ value = self.restore(v)
+ if (util.is_noncomplex(instance) or
+ util.is_dictionary_subclass(instance)):
+ instance[k] = value
+ else:
+ instance.__dict__[k] = value
+ # step out
+ self._namestack.pop()
# Handle list and set subclasses
if has_tag(obj, tags.SEQ):
diff --git a/src/setup.py b/src/setup.py
index 3bdfba2..20e4efd 100755
--- a/src/setup.py
+++ b/src/setup.py
@@ -8,25 +8,34 @@
# you should have received as part of this distribution.
+import sys
import jsonpickle as _jsonpickle
try:
from setuptools import setup
except ImportError:
from distutils.core import setup
-
-setup(
- name = "jsonpickle",
- version = _jsonpickle.__version__,
- description = "Python library for serializing any arbitrary object graph into JSON",
+
+
+INSTALL_REQUIRES = []
+if sys.version_info[:2] <= (2, 5):
+ INSTALL_REQUIRES.append('simplejson')
+
+
+SETUP_ARGS = dict(
+ name="jsonpickle",
+ version=_jsonpickle.__version__,
+ description="Python library for serializing any "
+ "arbitrary object graph into JSON",
long_description = _jsonpickle.__doc__,
- author = "John Paulett",
- author_email = "jo...@7oars.com",
- url = "http://code.google.com/p/jsonpickle/",
- license = "BSD",
- platforms = ['POSIX', 'Windows'],
- keywords = ['json pickle', 'json', 'pickle', 'marshal', 'serialization', 'JavaScript Object Notation'],
- classifiers = [
+ author="John Paulett",
+ author_email="jo...@7oars.com",
+ url="http://code.google.com/p/jsonpickle/",
+ license="BSD",
+ platforms=['POSIX', 'Windows'],
+ keywords=['json pickle', 'json', 'pickle', 'marshal',
+ 'serialization', 'JavaScript Object Notation'],
+ classifiers=[
"License :: OSI Approved :: BSD License",
"Operating System :: OS Independent",
"Programming Language :: Python",
@@ -35,12 +44,21 @@ setup(
"Intended Audience :: Developers",
"Programming Language :: JavaScript"
],
- options = { 'clean' : { 'all' : 1 } },
- packages = ["jsonpickle"],
- test_suite = 'jsonpickle.tests.suite',
- install_requires=["simplejson"],
+ options={'clean': {'all': 1}},
+ packages=["jsonpickle"],
+ test_suite='jsonpickle.tests.suite',
+ install_requires=INSTALL_REQUIRES,
extras_require = {
"cjson": ["python-cjson"]
},
zip_safe=True,
-)
\ No newline at end of file
+)
+
+
+def main():
+ setup(**SETUP_ARGS)
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())
\ No newline at end of file
--
1.6.0.4
David Aguilar
> diff --git a/src/benchmark.py b/src/benchmark.py
> index 866f287..f3a9676 100755
> --- a/src/benchmark.py
> +++ b/src/benchmark.py
Thanks, I've often overlooked benchmark.py.
> - def encode(self, obj):
> + def encode(self, obj, **kwargs):
I'm not sure if you noticed, but we already support
jsonpickle.set_encoder_options('json', indent=4)
jsonpickle.set_encoder_options('simplejson', indent=4)
The reason we do it that way is that the backend used might
not be the one you're expecting so we only apply the options
if that is indeed the current backend.
Passing the kwargs down unconditionally could in theory cause
a bug. I didn't think it was the nicest way of doing it, but
it's supported right now. Can you get away with doing it that
way instead?
> def decode(string):
> """Converts the JSON string into a Python object.
> diff --git a/src/jsonpickle/pickler.py b/src/jsonpickle/pickler.py
> index a8e1a78..0ee8807 100644
> --- a/src/jsonpickle/pickler.py
> +++ b/src/jsonpickle/pickler.py
> @@ -125,6 +125,7 @@ class Pickler(object):
> data = {}
> has_class = hasattr(obj, '__class__')
> has_dict = hasattr(obj, '__dict__')
> + has_getstate = hasattr(obj, '__getstate__')
I'm just finishing up on:
http://code.google.com/p/jsonpickle/issues/detail?id=7
which adds another has_slots thing.
As soon as that's done, I'll edit your patch and bring it
in. The __getstate__ stuff is useful indeed, and the logic is
sound.
This should be done by this weekend; likely within a few
hours.
Read on below for another note.
> diff --git a/src/jsonpickle/unpickler.py b/src/jsonpickle/unpickler.py
> index ae96685..edd3b72 100644
> --- a/src/jsonpickle/unpickler.py
> +++ b/src/jsonpickle/unpickler.py
> @@ -84,20 +84,24 @@ class Unpickler(object):
> # keep a obj->name mapping for use in the _isobjref() case
> self._mkref(instance)
>
> - for k, v in obj.iteritems():
> - # ignore the reserved attribute
> - if k in tags.RESERVED:
> - continue
> - self._namestack.append(k)
> - # step into the namespace
> - value = self.restore(v)
> - if (util.is_noncomplex(instance) or
> - util.is_dictionary_subclass(instance)):
> - instance[k] = value
> - else:
> - instance.__dict__[k] = value
> - # step out
> - self._namestack.pop()
> + if hasattr(instance, '__setstate__'):
> + instance.__setstate__(obj)
Hmm.
Do we not want to recurse into obj and restore it?
instance.__setstate__(self.restore(obj))
The symmetry of calling flatten() in the pickler and restore()
in the unpickler should probably be preserved here.
You probably didn't run into it since your __getstate__ /
__setstate__ functions probably work with simple
json-friendly lists. We don't need that restriction, though,
since things would stop working the moment __getstate__
started returning a dict with either a tuple, object instance,
or type ref, or some non-json-friendly data that jsonpickle
knows how to handle.
> diff --git a/src/setup.py b/src/setup.py
> index 3bdfba2..20e4efd 100755
> --- a/src/setup.py
> +++ b/src/setup.py
This looks good.
Can we get the setup.py stuff in a separate commit?
You might want to wait a couple of hours for the __getstate__
stuff to roll in.
dbuch
a separate commit, fixing the __setstate__ in Unpickler.
All has been pushed back to my fork at http://github.com/meatballhat/jsonpickle/tree/master
I'm also available by chat (via google/gmail), and will be happy to do
whatever is needed to get the __(get|set)state__ stuff merged in :)
> > - license = "BSD",
> > - platforms = ['POSIX', 'Windows'],
> > - keywords = ['json pickle', 'json', 'pickle', 'marshal', 'serialization', 'JavaScript Object Notation'],
> > - classifiers = [
> > + author="John Paulett",
David Aguilar
>
> I just did some more picking around to break the setup.py changes into
> a separate commit, fixing the __setstate__ in Unpickler.
Thanks for the good work everyone.
I've committed this stuff up to svn.
My git repo on github has the latest export from svn.
http://github.com/davvid/jsonpickle/tree/master
I did hand-edit a few things to make things match the naming of
the classes/vars/etc used in the __slots__ work, but aside from
that, all of the new functionality and fixes have been rolled
in.
When I committed it with git I was able to preserve
authorship information. git svn dcommit (or rather, svn),
unfortunately loses that stuff, though :-(
Dan, I did add a signed-off-by line in the commit messages with
your name on it.
All the tests are passing here. We have a few things cooking;
it seems like we're getting close to a new release.
John?
$ git log --pretty=format:%s tags/jsonpickle-0.2.0..
setup.py: Add a main() method to make setup.py import-safe
jsonpickle: Support objects with __setstate__() and __getstate__()
benchmark: Update to the latest jsonpickle API
pickler: Pre-fetch an object attribute to better support Zope
jsonpickle: Support newstyle classes with __slots__
jsonpickle.tests: Add a test for objects referencing submodules
jsonpickle.tests: Cleanup superfluous whitespace
jsonpickle: Add support for objects with module references
jsonpickle: add support for list and set subclasses
cleanup: scrub some whitespace, fix some typos, add some comments
tests: add more decode, remove_backend, and set_preferred_backend tests
pluginmgr: explicitly pass the decoder exception to callers
module init: add remove_backend to the jsonpickle namespace
pluginmgr: make load_backend() exception-safe
pickler: fortify jsonpickle against objects with broken __repr__s
BTW, I, too, prefer Unix line-endings, so I might make a pass
through the code and convert stuff over if that's okay with
everyone.
With the above commits, all issues except for #1 on the google
issue tracker have been closed as 'fixed'.
Just do a search for "Issues to verify" and you'll see that
issues 2 through 8 have been addressed.
I would like to hear back from the submitter of issue #6 since
I'm not exactly sure if I'm understanding exactly what they were
asking. I'm guessing at their email address so it might bounce
(or go to the wrong person ;)). Apologies in advance if that is
the case. Ditto for vollandt, d.halford, and otoomueller.
Here's a quick summary of the latest commits:
- Support __getstate__/__setstate__ ala the stdlib cPickle
module thanks to work by Dan Buch
- Properly handle new-style classes with __slots__;
issue reported by d.halford
- Support Zope persistent objects by pre-fetching the '_' attribute
before iterating over its __dict__ thanks to work by otoomueller
- Optimize serialization for objects with module references.
Some firepython objects were taking too long to serialize due
to objects with embedded module references. We now detect
modules and add inexpensive repr-like references instead of
trying to traverse into the module. Thanks to vollandt and
Antonin Hildebrand. I would like to get some feedback on this
one, though, as I only addressed objects-with-module-refs and
thus there might be some other performance tweaks that we'd
want to fold in.
Have fun and thanks for your help everyone,
--
David