Switch storage to cPickle to speed up processing

3 files changed, 19 insertions, 10 deletions

diff --git a/active_document/directory.py b/active_document/directory.py
index ee1db3d..4547c7b 100644
--- a/active_document/directory.py
+++ b/active_document/directory.py
@@ -27,7 +27,7 @@ from active_toolkit import util, enforce
 
 
 # To invalidate existed index on stcuture changes
-_LAYOUT_VERSION = 1
+_LAYOUT_VERSION = 3
 
 _GUID_RE = re.compile('[a-zA-Z0-9_+-.]+$')
 
@@ -66,7 +66,7 @@ class Directory(object):
             if exists(index_path):
                 _logger.warning('%r layout is stale, remove index',
                         self.metadata.name)
-            shutil.rmtree(index_path, ignore_errors=True)
+                shutil.rmtree(index_path, ignore_errors=True)
             self._save_layout()
 
         self._storage = Storage(root, self.metadata)
diff --git a/active_document/metadata.py b/active_document/metadata.py
index ec59a5b..42715dc 100644
--- a/active_document/metadata.py
+++ b/active_document/metadata.py
@@ -14,8 +14,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import json
 import types
+import cPickle as pickle
 from os.path import exists
 
 from active_document import env
@@ -113,7 +113,7 @@ class PropertyMeta(dict):
     def __init__(self, path_=None, **meta):
         if path_:
             with file(path_) as f:
-                meta.update(json.load(f))
+                meta.update(pickle.load(f))
             if exists(path_ + PropertyMeta.BLOB_SUFFIX):
                 meta['path'] = path_ + PropertyMeta.BLOB_SUFFIX
             meta['mtime'] = os.stat(path_).st_mtime
diff --git a/active_document/storage.py b/active_document/storage.py
index e24882b..0a20b23 100644
--- a/active_document/storage.py
+++ b/active_document/storage.py
@@ -19,6 +19,7 @@ import time
 import json
 import shutil
 import hashlib
+import cPickle as pickle
 from base64 import b64decode
 from os.path import exists, join, isdir, basename, relpath, lexists, isabs
 
@@ -95,16 +96,24 @@ class Storage(object):
         if exists(path):
             seqno = int(os.stat(path).st_mtime)
             with file(join(root, 'seqno'), 'w') as f:
-                json.dump({'seqno': seqno, 'value': seqno}, f)
+                pickle.dump({'seqno': seqno, 'value': seqno}, f)
             os.unlink(path)
 
         for name, prop in self.metadata.items():
             path = join(root, name)
             if exists(path + '.seqno'):
                 self._migrate_to_1(path, prop)
-            elif not exists(path):
-                if not isinstance(prop, BlobProperty):
-                    record.set(name, seqno=0, value=prop.default)
+                continue
+            if exists(path):
+                with file(path) as f:
+                    meta = f.read()
+                if meta:
+                    if meta[0] == '{':
+                        with file(path, 'w') as f:
+                            pickle.dump(json.loads(meta), f)
+                    continue
+            if not isinstance(prop, BlobProperty) and prop.default is not None:
+                record.set(name, seqno=0, value=prop.default)
 
     def _migrate_to_1(self, path, prop):
         meta = {'seqno': int(os.stat(path + '.seqno').st_mtime)}
@@ -143,7 +152,7 @@ class Storage(object):
 
         if meta is not None:
             with file(path, 'w') as f:
-                json.dump(meta, f)
+                pickle.dump(meta, f)
             if mtime is not None:
                 os.utime(path, (mtime, mtime))
 
@@ -194,7 +203,7 @@ class Record(object):
             util.cptree(path, blob_path)
 
         with util.new_file(meta_path) as f:
-            json.dump(meta, f)
+            pickle.dump(meta, f)
         if mtime:
             os.utime(meta_path, (mtime, mtime))