Switch storage to cPickle to speed up processing

8 files changed, 59 insertions, 35 deletions

diff --git a/active_document/directory.py b/active_document/directory.py
index ee1db3d..4547c7b 100644
--- a/active_document/directory.py
+++ b/active_document/directory.py
@@ -27,7 +27,7 @@ from active_toolkit import util, enforce
 
 
 # To invalidate existed index on stcuture changes
-_LAYOUT_VERSION = 1
+_LAYOUT_VERSION = 3
 
 _GUID_RE = re.compile('[a-zA-Z0-9_+-.]+$')
 
@@ -66,7 +66,7 @@ class Directory(object):
             if exists(index_path):
                 _logger.warning('%r layout is stale, remove index',
                         self.metadata.name)
-            shutil.rmtree(index_path, ignore_errors=True)
+                shutil.rmtree(index_path, ignore_errors=True)
             self._save_layout()
 
         self._storage = Storage(root, self.metadata)
diff --git a/active_document/metadata.py b/active_document/metadata.py
index ec59a5b..42715dc 100644
--- a/active_document/metadata.py
+++ b/active_document/metadata.py
@@ -14,8 +14,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import json
 import types
+import cPickle as pickle
 from os.path import exists
 
 from active_document import env
@@ -113,7 +113,7 @@ class PropertyMeta(dict):
     def __init__(self, path_=None, **meta):
         if path_:
             with file(path_) as f:
-                meta.update(json.load(f))
+                meta.update(pickle.load(f))
             if exists(path_ + PropertyMeta.BLOB_SUFFIX):
                 meta['path'] = path_ + PropertyMeta.BLOB_SUFFIX
             meta['mtime'] = os.stat(path_).st_mtime
diff --git a/active_document/storage.py b/active_document/storage.py
index e24882b..0a20b23 100644
--- a/active_document/storage.py
+++ b/active_document/storage.py
@@ -19,6 +19,7 @@ import time
 import json
 import shutil
 import hashlib
+import cPickle as pickle
 from base64 import b64decode
 from os.path import exists, join, isdir, basename, relpath, lexists, isabs
 
@@ -95,16 +96,24 @@ class Storage(object):
         if exists(path):
             seqno = int(os.stat(path).st_mtime)
             with file(join(root, 'seqno'), 'w') as f:
-                json.dump({'seqno': seqno, 'value': seqno}, f)
+                pickle.dump({'seqno': seqno, 'value': seqno}, f)
             os.unlink(path)
 
         for name, prop in self.metadata.items():
             path = join(root, name)
             if exists(path + '.seqno'):
                 self._migrate_to_1(path, prop)
-            elif not exists(path):
-                if not isinstance(prop, BlobProperty):
-                    record.set(name, seqno=0, value=prop.default)
+                continue
+            if exists(path):
+                with file(path) as f:
+                    meta = f.read()
+                if meta:
+                    if meta[0] == '{':
+                        with file(path, 'w') as f:
+                            pickle.dump(json.loads(meta), f)
+                    continue
+            if not isinstance(prop, BlobProperty) and prop.default is not None:
+                record.set(name, seqno=0, value=prop.default)
 
     def _migrate_to_1(self, path, prop):
         meta = {'seqno': int(os.stat(path + '.seqno').st_mtime)}
@@ -143,7 +152,7 @@ class Storage(object):
 
         if meta is not None:
             with file(path, 'w') as f:
-                json.dump(meta, f)
+                pickle.dump(meta, f)
             if mtime is not None:
                 os.utime(path, (mtime, mtime))
 
@@ -194,7 +203,7 @@ class Record(object):
             util.cptree(path, blob_path)
 
         with util.new_file(meta_path) as f:
-            json.dump(meta, f)
+            pickle.dump(meta, f)
         if mtime:
             os.utime(meta_path, (mtime, mtime))
 
diff --git a/tests/units/document.py b/tests/units/document.py
index 879c5ea..3951daa 100755
--- a/tests/units/document.py
+++ b/tests/units/document.py
@@ -5,10 +5,10 @@
 import os
 import sys
 import stat
-import json
 import time
 import urllib2
 import hashlib
+import cPickle as pickle
 from base64 import b64encode
 from cStringIO import StringIO
 from os.path import join, exists
@@ -325,12 +325,12 @@ class DocumentTest(tests.Test):
         populated = 0
         for i in directory.populate():
             populated += 1
-        self.assertEqual(1, populated)
+        self.assertEqual(2, populated)
         self.assertEqual(
-                ['1'],
-                [i.guid for i in directory.find(0, 10)[0]])
+                sorted(['1', '2']),
+                sorted([i.guid for i in directory.find(0, 10)[0]]))
         assert exists('1/1/guid')
-        assert not exists('2/2/guid')
+        assert exists('2/2/guid')
         assert not exists('3/3/guid')
 
     def test_create_with_guid(self):
@@ -371,46 +371,46 @@ class DocumentTest(tests.Test):
         seqno = directory.get(guid_1).get('seqno')
         self.assertEqual(1, seqno)
         self.assertEqual(
-                json.load(file('%s/%s/guid' % (guid_1[:2], guid_1)))['seqno'],
+                pickle.load(file('%s/%s/guid' % (guid_1[:2], guid_1)))['seqno'],
                 seqno)
         self.assertEqual(
-                json.load(file('%s/%s/prop' % (guid_1[:2], guid_1)))['seqno'],
+                pickle.load(file('%s/%s/prop' % (guid_1[:2], guid_1)))['seqno'],
                 seqno)
 
         guid_2 = directory.create({})
         seqno = directory.get(guid_2).get('seqno')
         self.assertEqual(2, seqno)
         self.assertEqual(
-                json.load(file('%s/%s/guid' % (guid_2[:2], guid_2)))['seqno'],
+                pickle.load(file('%s/%s/guid' % (guid_2[:2], guid_2)))['seqno'],
                 seqno)
         self.assertEqual(
-                json.load(file('%s/%s/prop' % (guid_2[:2], guid_2)))['seqno'],
+                pickle.load(file('%s/%s/prop' % (guid_2[:2], guid_2)))['seqno'],
                 seqno)
 
         directory.set_blob(guid_1, 'blob', StringIO('blob'))
         seqno = directory.get(guid_1).get('seqno')
         self.assertEqual(3, seqno)
         self.assertEqual(
-                json.load(file('%s/%s/guid' % (guid_1[:2], guid_1)))['seqno'],
+                pickle.load(file('%s/%s/guid' % (guid_1[:2], guid_1)))['seqno'],
                 1)
         self.assertEqual(
-                json.load(file('%s/%s/prop' % (guid_1[:2], guid_1)))['seqno'],
+                pickle.load(file('%s/%s/prop' % (guid_1[:2], guid_1)))['seqno'],
                 1)
         self.assertEqual(
-                json.load(file('%s/%s/blob' % (guid_1[:2], guid_1)))['seqno'],
+                pickle.load(file('%s/%s/blob' % (guid_1[:2], guid_1)))['seqno'],
                 seqno)
 
         directory.update(guid_1, {'prop': 'new'})
         seqno = directory.get(guid_1).get('seqno')
         self.assertEqual(4, seqno)
         self.assertEqual(
-                json.load(file('%s/%s/guid' % (guid_1[:2], guid_1)))['seqno'],
+                pickle.load(file('%s/%s/guid' % (guid_1[:2], guid_1)))['seqno'],
                 1)
         self.assertEqual(
-                json.load(file('%s/%s/prop' % (guid_1[:2], guid_1)))['seqno'],
+                pickle.load(file('%s/%s/prop' % (guid_1[:2], guid_1)))['seqno'],
                 seqno)
         self.assertEqual(
-                json.load(file('%s/%s/blob' % (guid_1[:2], guid_1)))['seqno'],
+                pickle.load(file('%s/%s/blob' % (guid_1[:2], guid_1)))['seqno'],
                 3)
 
     def test_diff(self):
diff --git a/tests/units/metadata.py b/tests/units/metadata.py
index 2251e14..37058c9 100755
--- a/tests/units/metadata.py
+++ b/tests/units/metadata.py
@@ -1,8 +1,6 @@
 #!/usr/bin/env python
 # sugar-lint: disable
 
-import json
-
 from __init__ import tests
 
 from active_document import metadata
diff --git a/tests/units/migrate.py b/tests/units/migrate.py
index 4629a1f..ef3cd94 100755
--- a/tests/units/migrate.py
+++ b/tests/units/migrate.py
@@ -255,17 +255,35 @@ class MigrateTest(tests.Test):
         for i in directory.populate():
             pass
 
-        assert exists('gu/guid/prop1')
+        assert not exists('gu/guid/prop1')
         assert exists('gu/guid/prop2')
 
         doc = directory.get('guid')
         self.assertEqual(
-                {'value': None, 'seqno': 0, 'mtime': os.stat('gu/guid/prop1').st_mtime},
-                doc.meta('prop1'))
-        self.assertEqual(
-                {'value': 'default', 'seqno': 0, 'mtime': os.stat('gu/guid/prop1').st_mtime},
+                {'value': 'default', 'seqno': 0, 'mtime': os.stat('gu/guid/prop2').st_mtime},
                 doc.meta('prop2'))
 
+    def test_ConvertFromJson(self):
+
+        class Document(document.Document):
+
+            @active_property(prefix='P', default='value')
+            def prop(self, value):
+                return value
+
+        guid_value = '{"value": "guid"}'
+        self.touch(('gu/guid/guid', guid_value))
+
+        directory = Directory(tests.tmpdir, Document, IndexWriter)
+        for i in directory.populate():
+            pass
+
+        doc = directory.get('guid')
+        self.assertEqual(
+                {'value': 'guid', 'mtime': os.stat('gu/guid/guid').st_mtime},
+                doc.meta('guid'))
+        self.assertNotEqual(guid_value, file('gu/guid/guid').read())
+
 
 if __name__ == '__main__':
     tests.main()
diff --git a/tests/units/storage.py b/tests/units/storage.py
index dc87309..60adcc2 100755
--- a/tests/units/storage.py
+++ b/tests/units/storage.py
@@ -3,9 +3,9 @@
 
 import os
 import time
-import json
 import hashlib
 import threading
+import cPickle as pickle
 from cStringIO import StringIO
 from os.path import exists
 
@@ -35,7 +35,7 @@ class StorageTest(tests.Test):
         storage = self.storage([StoredProperty('prop')])
 
         self.assertEqual(None, storage.get('guid').get('prop'))
-        self.touch(('gu/guid/prop', json.dumps({
+        self.touch(('gu/guid/prop', pickle.dumps({
             'value': 'value',
             'foo': 'bar',
             })))
diff --git a/tests/units/volume.py b/tests/units/volume.py
index 777223d..7ca8d2e 100755
--- a/tests/units/volume.py
+++ b/tests/units/volume.py
@@ -4,7 +4,6 @@
 import os
 import sys
 import time
-import json
 import shutil
 import hashlib
 from cStringIO import StringIO