Merge branch 'upstream-git' into upstream

4 files changed, 256 insertions, 26 deletions

diff --git a/configure.ac b/configure.ac
index bd353b1..07d00a7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,4 +1,4 @@
-AC_INIT([sugar-datastore],[0.8.1],[],[sugar-datastore])
+AC_INIT([sugar-datastore],[0.8.2],[],[sugar-datastore])
 
 AC_PREREQ([2.59])
 
diff --git a/src/olpc/datastore/backingstore.py b/src/olpc/datastore/backingstore.py
index fc3c05f..cd23680 100644
--- a/src/olpc/datastore/backingstore.py
+++ b/src/olpc/datastore/backingstore.py
@@ -29,6 +29,13 @@ import dbus
 import xapian
 import gobject
 
+try:
+    import cjson
+    has_cjson = True
+except ImportError:
+    import simplejson
+    has_cjson = False
+
 from olpc.datastore.xapianindex import IndexManager
 from olpc.datastore import bin_copy
 from olpc.datastore import utils
@@ -215,7 +222,11 @@ class FileBackingStore(BackingStore):
         instead of a method parameter because this is less invasive for Update 1.
         """
         self.current_user_id = None
-        
+
+        # source for an idle callback that exports to the file system the
+        # metadata from the index
+        self._export_metadata_source = None
+
     # Informational
     def descriptor(self):
         """return a dict with atleast the following keys
@@ -327,7 +338,28 @@ class FileBackingStore(BackingStore):
             im.connect(index_name)
 
             self.indexmanager = im
-            
+
+        # Check that all entries have their metadata in the file system.
+        if not os.path.exists(os.path.join(self.base, '.metadata.exported')):
+            uids_to_export = []
+            uids = self.indexmanager.get_all_ids()
+
+            for uid in uids:
+                if not os.path.exists(os.path.join(self.base, uid + '.metadata')):
+                    uids_to_export.append(uid)
+
+            if uids_to_export:
+                self._export_metadata_source = gobject.idle_add(
+                        self._export_metadata, uids_to_export)
+            else:
+                open(os.path.join(self.base, '.metadata.exported'), 'w').close()
+
+    def _export_metadata(self, uids_to_export):
+        uid = uids_to_export.pop()
+        props = self.indexmanager.get(uid).properties
+        self._store_metadata(uid, props)
+        return len(uids_to_export) > 0
+
     def bind_to(self, datastore):
         ## signal from datastore that we are being bound to it
         self.datastore = datastore
@@ -500,8 +532,33 @@ class FileBackingStore(BackingStore):
             c.update(line)
         fp.close()
         return c.hexdigest()
-        
+
     # File Management API
+    def _encode_json(self, metadata, file_path):
+        if has_cjson:
+            f = open(file_path, 'w')
+            f.write(cjson.encode(metadata))
+            f.close()
+        else:
+            simplejson.dump(metadata, open(file_path, 'w'))
+
+    def _store_metadata(self, uid, props):
+        t = time.time()
+        temp_path = os.path.join(self.base, '.temp_metadata')
+        props = props.copy()
+        for property_name in model.defaultModel.get_external_properties():
+            if property_name in props:
+                del props[property_name]
+        self._encode_json(props, temp_path)
+        path = os.path.join(self.base, uid + '.metadata')
+        os.rename(temp_path, path)
+        logging.debug('exported metadata: %r s.' % (time.time() - t))
+
+    def _delete_metadata(self, uid):
+        path = os.path.join(self.base, uid + '.metadata')
+        if os.path.exists(path):
+            os.unlink(path)
+
     def _create_completion(self, uid, props, completion, exc=None, path=None):
         if exc:
             completion(exc)
@@ -517,6 +574,7 @@ class FileBackingStore(BackingStore):
         if completion is None:
             raise RuntimeError("Completion must be valid for async create")
         uid = self.indexmanager.index(props)
+        self._store_metadata(uid, props)
         props['uid'] = uid
         if filelike:
             if isinstance(filelike, basestring):
@@ -531,6 +589,7 @@ class FileBackingStore(BackingStore):
     def create(self, props, filelike, can_move=False):
         if filelike:
             uid = self.indexmanager.index(props)
+            self._store_metadata(uid, props)
             props['uid'] = uid
             if isinstance(filelike, basestring):
                 # lets treat it as a filename
@@ -540,7 +599,9 @@ class FileBackingStore(BackingStore):
             self.indexmanager.index(props, path)
             return uid
         else:
-            return self.indexmanager.index(props)
+            uid = self.indexmanager.index(props)
+            self._store_metadata(uid, props)
+            return uid
     
     def get(self, uid, env=None, allowMissing=False, includeFile=False):
         content = self.indexmanager.get(uid)
@@ -575,6 +636,7 @@ class FileBackingStore(BackingStore):
             raise RuntimeError("Completion must be valid for async update")
 
         props['uid'] = uid
+        self._store_metadata(uid, props)
         if filelike:
             uid = self.indexmanager.index(props, filelike)
             props['uid'] = uid
@@ -590,6 +652,7 @@ class FileBackingStore(BackingStore):
 
     def update(self, uid, props, filelike=None, can_move=False):
         props['uid'] = uid
+        self._store_metadata(uid, props)
         if filelike:
             if isinstance(filelike, basestring):
                 # lets treat it as a filename
@@ -610,6 +673,7 @@ class FileBackingStore(BackingStore):
 
     def delete(self, uid, allowMissing=True):
         self._delete_external_properties(uid)
+        self._delete_metadata(uid)
 
         self.indexmanager.delete(uid)
         path = self._translatePath(uid)
@@ -617,7 +681,7 @@ class FileBackingStore(BackingStore):
             os.unlink(path)
         else:
             if not allowMissing:
-                raise KeyError("object for uid:%s missing" % uid)            
+                raise KeyError("object for uid:%s missing" % uid)
         
     def get_uniquevaluesfor(self, propertyname):
         return self.indexmanager.get_uniquevaluesfor(propertyname)
@@ -651,6 +715,8 @@ class FileBackingStore(BackingStore):
         return self.indexmanager.get_all_ids()
     
     def stop(self):
+        if self._export_metadata_source is not None:
+            gobject.source_remove(self._export_metadata_source)
         self.indexmanager.stop()
 
     def complete_indexing(self):
diff --git a/src/olpc/datastore/datastore.py b/src/olpc/datastore/datastore.py
index 67ddca9..a15d5cf 100644
--- a/src/olpc/datastore/datastore.py
+++ b/src/olpc/datastore/datastore.py
@@ -128,28 +128,10 @@ class DataStore(dbus.service.Object):
 
     ### Backup support
     def pause(self, mountpoints=None):
-        """pause the datastore, during this time it will not process
-    requests. this allows the underlying stores to be backup up via
-    traditional mechanisms
-    """
-        if mountpoints:
-            mps = [self.mountpoints[mp] for mp in mountpoints]
-        else:
-            mps = self.mountpoints.values()
-
-        for mp in mps:
-            mp.stop()
+        """ Deprecated. """
 
     def unpause(self, mountpoints=None):
-        """resume the operation of a set of paused mountpoints"""
-        if mountpoints:
-            mps = [self.mountpoints[mp] for mp in mountpoints]
-        else:
-            mps = self.mountpoints.values()
-
-        for mp in mps:
-            mp.initialize_and_load()
-        
+        """ Deprecated. """
     ### End Backups
             
     def connect_backingstore(self, uri, **kwargs):
diff --git a/tests/test_sugar.py b/tests/test_sugar.py
new file mode 100644
index 0000000..39a0e8c
--- /dev/null
+++ b/tests/test_sugar.py
@@ -0,0 +1,182 @@
+# Copyright (C) 2007 One Laptop Per Child
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+""" These tests try to cover how the DS is being used inside Sugar.
+"""
+
+import sys
+import os
+import unittest
+import time
+import tempfile
+import shutil
+from datetime import datetime
+
+import dbus
+
+DS_DBUS_SERVICE = "org.laptop.sugar.DataStore"
+DS_DBUS_INTERFACE = "org.laptop.sugar.DataStore"
+DS_DBUS_PATH = "/org/laptop/sugar/DataStore"
+
+PROPS_WITHOUT_PREVIEW = {'activity_id': '37fa2f4013b17ae7fc6448f10fe5df53ef92de18',
+        'title_set_by_user': '0',
+        'title': 'Write Activity',
+        'timestamp': int(time.time()),
+        'activity': 'org.laptop.AbiWordActivity',
+        'share-scope': 'private',
+        'keep': '0',
+        'icon-color': '#00588C,#00EA11',
+        'mtime': datetime.now().isoformat(),
+        'preview': '',
+        'mime_type': ''}
+
+PROPS_WITH_PREVIEW = {'activity_id': 'e8594bea74faa80539d93ef1a10de3c712bb2eac',
+        'title_set_by_user': '0',
+        'title': 'Write Activity',
+        'share-scope': 'private',
+        'timestamp': int(time.time()),
+        'activity': 'org.laptop.AbiWordActivity',
+        'fulltext': 'mec mac',
+        'keep': '0',
+        'icon-color': '#00588C,#00EA11',
+        'mtime': datetime.now().isoformat(), 
+        'preview': dbus.ByteArray('\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\xd8\x00\x00\x00\xa2\x08\x02\x00\x00\x00\xac\xfb\x94\x1d\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x03\x1dIDATx\x9c\xed\xd6\xbfJci\x00\xc6\xe1\xc4\xb5Qb\xb0Qa\xfc\xb3\x0cV\xda\xc8,\xa4\xf0^,\xbc;\xef@\x04kA\xb3(\x16b\x97 3("\xc9\x14\'\x13\xc5\x9c\x9cq\x8bmg\xdcj\x93\x17\xf3<\xedw\x8a\xf7\x83\x1f\xe7\x9c\xfa\xd1\xd1\xd1\xc6\xc6F\r\xa6\xe4\xf1\xf1\xf1\xfe\xfe~~{{\xbb\xd5jM{\x0c\xb3\xeb\xe1\xe1\xa1\xddn\xcf\xbf\xf3D\xaf\xd7\xbb\xbd\xbdm4\x1aooo\xbb\xbb\xbb\x97\x97\x97;;;\xa3\xd1\xa8\xd3\xe9\xb4Z\xad\xaa\xaa...\x96\x96\x96\xca\xb2\xdc\xdf\xdf???_[[[^^\xbe\xb9\xb9\xd9\xdb\xdbk6\x9b\x13\xbb\t\x1f\xc0{!\xd6\xeb\xf5n\xb7\xfb\xf4\xf4\xb4\xb0\xb0p}}\xbd\xbe\xbe~rrR\x14\xc5\xc1\xc1AY\x96\x8dF\xe3\xea\xea\xaa\xaa\xaa\xc5\xc5\xc5V\xab\xd5\xe9tNOOWVV\x0e\x0f\x0f\x87\xc3\xe1\xc4.\xc0\xc70\xf7\xce\xd9`0(\xcb\xf2\xc7`\xf0\xbd\xdf\xffsk\xebgU\xfd\xf5\xe5K\xb3\xd9\xfc\xbb\xdd\xfecn\xee\xf9\xf9\xf9\xf5\xf5\xb5(\x8a\xe1pxww\xd7\xef\xf7\x8b\xa2X]]=;;\x9b\xd8z>\x8cz\xbb\xdd\xfe\xed?\xe2\xb7o\xb5\xaf_\x7f}\xf4\xe9S\xed\xf3\xe7\xffo\x16\xb3\xe3\xbf\xff\x11k\x9b\x9b\xb5\xcd\xcdI\xeda\xa6\xbd\xf7i\x86\x89\x11"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!2e\xe3\xf1x<\x1e\xcfO{\x06\xb3\xee\xf8\xf8\xb8\xd7\xeby#2e///UU\t\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\xccw\xbb\xdd\xd1h4\xed\x19\xcc\xae\x7f\xf3\xfb\x07q8\x9emk8\x97\xda\x00\x00\x00\x00IEND\xaeB`\x82'),
+        'mime_type': 'application/vnd.oasis.opendocument.text'}
+
+class CommonTest(unittest.TestCase):
+
+    def setUp(self):
+        bus = dbus.SessionBus()
+        proxy = bus.get_object(DS_DBUS_SERVICE, DS_DBUS_PATH)
+        self._data_store = dbus.Interface(proxy, DS_DBUS_INTERFACE)
+
+    def create(self):
+        file_path = self._prepare_file()
+        
+        t = time.time()
+        uid = self._data_store.create(PROPS_WITHOUT_PREVIEW, file_path, True)
+        t = time.time() - t
+        return t, uid
+
+    def update(self, uid):
+        file_path = self._prepare_file()
+        t = time.time()
+        self._data_store.update(uid, PROPS_WITH_PREVIEW, file_path, True)
+        t = time.time() - t
+        return t
+
+    def find(self):
+        query = {'order_by': ['-mtime'],
+                 'limit': 80}
+        t = time.time()
+        results, count = self._data_store.find(query, [])
+        t = time.time() - t
+        return t
+
+    def _prepare_file(self):
+        file_path = os.path.join(os.getcwd(), 'tests/funkyabi.odt')
+        f, tmp_path = tempfile.mkstemp()
+        os.close(f)
+        shutil.copyfile(file_path, tmp_path)
+        return tmp_path
+
+class FunctionalityTest(CommonTest):
+
+    def testcreation(self):
+        t, uid = self.create()
+        assert uid
+
+    def testupdate(self):
+        t, uid = self.create()
+        t = self.update(uid)
+
+    def testresume(self):
+        t, uid = self.create()
+        props = self._data_store.get_properties(uid, byte_arrays=True)
+        #del props['uid']
+        assert props == PROPS_WITHOUT_PREVIEW
+
+        t = self.update(uid)
+        props = self._data_store.get_properties(uid, byte_arrays=True)
+        #del props['uid']
+        assert props == PROPS_WITH_PREVIEW
+
+        file_name = self._data_store.get_filename(uid)
+
+        assert os.path.exists(file_name)
+        f = open(file_name, 'r')
+        f.close()
+
+    """
+    def testcustomproperties(self):
+        t, uid = self.create()
+        props = self._data_store.get_properties(uid)
+        props['custom_property'] = 'test'
+        self._data_store.update(uid, props, '', True)
+
+        props = self._data_store.get_properties(uid)
+        assert props['custom_property'] == 'test'
+
+        results, count = self._data_store.find({'custom_property': 'test'}, [])
+        assert count > 1
+        for entry in results:
+            assert entry['custom_property'] == 'test'
+            uid = entry['uid']
+            props = self._data_store.get_properties(uid)
+            assert props['custom_property'] == 'test'
+    """
+
+    def testfind(self):
+        t = self.find()
+
+class PerformanceTest(CommonTest):
+
+    def _avg(self, l):
+        total = 0
+        for i in l:
+            total += i
+        return total / len(l)
+
+    def _test_perf(self, label, function, iterations):    
+        t_max = 0
+        t_min = sys.maxint
+        times = []
+        for i in range(1, iterations):
+            t = function()
+            t_max = max(t, t_max)
+            t_min = min(t, t_min)
+            times.append(t)
+            
+        print '%s max: %.3fms min: %.3fms avg: %.3fms' % \
+                (label, t_max * 1000, t_min * 1000, self._avg(times) * 1000)
+
+    def testperformance(self):
+        iterations = 100
+
+        self._test_perf('Create', lambda: self.create()[0], iterations)
+
+        t, uid = self.create()
+        self._test_perf('Update', lambda: self.update(uid), iterations)
+
+        self._test_perf('Find', lambda: self.find(), iterations)
+
+
+if __name__ == '__main__':
+    suite = unittest.TestSuite()
+    #suite.addTest(unittest.makeSuite(FunctionalityTest))
+    suite.addTest(unittest.makeSuite(PerformanceTest))
+    unittest.TextTestRunner().run(suite)
+