From 43a4981aeaaabe39b8261d55415fa18a2a2d74c9 Mon Sep 17 00:00:00 2001 From: Jonas Smedegaard Date: Fri, 20 Jun 2008 23:02:44 +0000 Subject: Merge branch 'upstream-git' into upstream --- diff --git a/configure.ac b/configure.ac index bd353b1..07d00a7 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([sugar-datastore],[0.8.1],[],[sugar-datastore]) +AC_INIT([sugar-datastore],[0.8.2],[],[sugar-datastore]) AC_PREREQ([2.59]) diff --git a/src/olpc/datastore/backingstore.py b/src/olpc/datastore/backingstore.py index fc3c05f..cd23680 100644 --- a/src/olpc/datastore/backingstore.py +++ b/src/olpc/datastore/backingstore.py @@ -29,6 +29,13 @@ import dbus import xapian import gobject +try: + import cjson + has_cjson = True +except ImportError: + import simplejson + has_cjson = False + from olpc.datastore.xapianindex import IndexManager from olpc.datastore import bin_copy from olpc.datastore import utils @@ -215,7 +222,11 @@ class FileBackingStore(BackingStore): instead of a method parameter because this is less invasive for Update 1. """ self.current_user_id = None - + + # source for an idle callback that exports to the file system the + # metadata from the index + self._export_metadata_source = None + # Informational def descriptor(self): """return a dict with atleast the following keys @@ -327,7 +338,28 @@ class FileBackingStore(BackingStore): im.connect(index_name) self.indexmanager = im - + + # Check that all entries have their metadata in the file system. + if not os.path.exists(os.path.join(self.base, '.metadata.exported')): + uids_to_export = [] + uids = self.indexmanager.get_all_ids() + + for uid in uids: + if not os.path.exists(os.path.join(self.base, uid + '.metadata')): + uids_to_export.append(uid) + + if uids_to_export: + self._export_metadata_source = gobject.idle_add( + self._export_metadata, uids_to_export) + else: + open(os.path.join(self.base, '.metadata.exported'), 'w').close() + + def _export_metadata(self, uids_to_export): + uid = uids_to_export.pop() + props = self.indexmanager.get(uid).properties + self._store_metadata(uid, props) + return len(uids_to_export) > 0 + def bind_to(self, datastore): ## signal from datastore that we are being bound to it self.datastore = datastore @@ -500,8 +532,33 @@ class FileBackingStore(BackingStore): c.update(line) fp.close() return c.hexdigest() - + # File Management API + def _encode_json(self, metadata, file_path): + if has_cjson: + f = open(file_path, 'w') + f.write(cjson.encode(metadata)) + f.close() + else: + simplejson.dump(metadata, open(file_path, 'w')) + + def _store_metadata(self, uid, props): + t = time.time() + temp_path = os.path.join(self.base, '.temp_metadata') + props = props.copy() + for property_name in model.defaultModel.get_external_properties(): + if property_name in props: + del props[property_name] + self._encode_json(props, temp_path) + path = os.path.join(self.base, uid + '.metadata') + os.rename(temp_path, path) + logging.debug('exported metadata: %r s.' % (time.time() - t)) + + def _delete_metadata(self, uid): + path = os.path.join(self.base, uid + '.metadata') + if os.path.exists(path): + os.unlink(path) + def _create_completion(self, uid, props, completion, exc=None, path=None): if exc: completion(exc) @@ -517,6 +574,7 @@ class FileBackingStore(BackingStore): if completion is None: raise RuntimeError("Completion must be valid for async create") uid = self.indexmanager.index(props) + self._store_metadata(uid, props) props['uid'] = uid if filelike: if isinstance(filelike, basestring): @@ -531,6 +589,7 @@ class FileBackingStore(BackingStore): def create(self, props, filelike, can_move=False): if filelike: uid = self.indexmanager.index(props) + self._store_metadata(uid, props) props['uid'] = uid if isinstance(filelike, basestring): # lets treat it as a filename @@ -540,7 +599,9 @@ class FileBackingStore(BackingStore): self.indexmanager.index(props, path) return uid else: - return self.indexmanager.index(props) + uid = self.indexmanager.index(props) + self._store_metadata(uid, props) + return uid def get(self, uid, env=None, allowMissing=False, includeFile=False): content = self.indexmanager.get(uid) @@ -575,6 +636,7 @@ class FileBackingStore(BackingStore): raise RuntimeError("Completion must be valid for async update") props['uid'] = uid + self._store_metadata(uid, props) if filelike: uid = self.indexmanager.index(props, filelike) props['uid'] = uid @@ -590,6 +652,7 @@ class FileBackingStore(BackingStore): def update(self, uid, props, filelike=None, can_move=False): props['uid'] = uid + self._store_metadata(uid, props) if filelike: if isinstance(filelike, basestring): # lets treat it as a filename @@ -610,6 +673,7 @@ class FileBackingStore(BackingStore): def delete(self, uid, allowMissing=True): self._delete_external_properties(uid) + self._delete_metadata(uid) self.indexmanager.delete(uid) path = self._translatePath(uid) @@ -617,7 +681,7 @@ class FileBackingStore(BackingStore): os.unlink(path) else: if not allowMissing: - raise KeyError("object for uid:%s missing" % uid) + raise KeyError("object for uid:%s missing" % uid) def get_uniquevaluesfor(self, propertyname): return self.indexmanager.get_uniquevaluesfor(propertyname) @@ -651,6 +715,8 @@ class FileBackingStore(BackingStore): return self.indexmanager.get_all_ids() def stop(self): + if self._export_metadata_source is not None: + gobject.source_remove(self._export_metadata_source) self.indexmanager.stop() def complete_indexing(self): diff --git a/src/olpc/datastore/datastore.py b/src/olpc/datastore/datastore.py index 67ddca9..a15d5cf 100644 --- a/src/olpc/datastore/datastore.py +++ b/src/olpc/datastore/datastore.py @@ -128,28 +128,10 @@ class DataStore(dbus.service.Object): ### Backup support def pause(self, mountpoints=None): - """pause the datastore, during this time it will not process - requests. this allows the underlying stores to be backup up via - traditional mechanisms - """ - if mountpoints: - mps = [self.mountpoints[mp] for mp in mountpoints] - else: - mps = self.mountpoints.values() - - for mp in mps: - mp.stop() + """ Deprecated. """ def unpause(self, mountpoints=None): - """resume the operation of a set of paused mountpoints""" - if mountpoints: - mps = [self.mountpoints[mp] for mp in mountpoints] - else: - mps = self.mountpoints.values() - - for mp in mps: - mp.initialize_and_load() - + """ Deprecated. """ ### End Backups def connect_backingstore(self, uri, **kwargs): diff --git a/tests/test_sugar.py b/tests/test_sugar.py new file mode 100644 index 0000000..39a0e8c --- /dev/null +++ b/tests/test_sugar.py @@ -0,0 +1,182 @@ +# Copyright (C) 2007 One Laptop Per Child +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +""" These tests try to cover how the DS is being used inside Sugar. +""" + +import sys +import os +import unittest +import time +import tempfile +import shutil +from datetime import datetime + +import dbus + +DS_DBUS_SERVICE = "org.laptop.sugar.DataStore" +DS_DBUS_INTERFACE = "org.laptop.sugar.DataStore" +DS_DBUS_PATH = "/org/laptop/sugar/DataStore" + +PROPS_WITHOUT_PREVIEW = {'activity_id': '37fa2f4013b17ae7fc6448f10fe5df53ef92de18', + 'title_set_by_user': '0', + 'title': 'Write Activity', + 'timestamp': int(time.time()), + 'activity': 'org.laptop.AbiWordActivity', + 'share-scope': 'private', + 'keep': '0', + 'icon-color': '#00588C,#00EA11', + 'mtime': datetime.now().isoformat(), + 'preview': '', + 'mime_type': ''} + +PROPS_WITH_PREVIEW = {'activity_id': 'e8594bea74faa80539d93ef1a10de3c712bb2eac', + 'title_set_by_user': '0', + 'title': 'Write Activity', + 'share-scope': 'private', + 'timestamp': int(time.time()), + 'activity': 'org.laptop.AbiWordActivity', + 'fulltext': 'mec mac', + 'keep': '0', + 'icon-color': '#00588C,#00EA11', + 'mtime': datetime.now().isoformat(), + 'preview': dbus.ByteArray('\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\xd8\x00\x00\x00\xa2\x08\x02\x00\x00\x00\xac\xfb\x94\x1d\x00\x00\x00\x03sBIT\x08\x08\x08\xdb\xe1O\xe0\x00\x00\x03\x1dIDATx\x9c\xed\xd6\xbfJci\x00\xc6\xe1\xc4\xb5Qb\xb0Qa\xfc\xb3\x0cV\xda\xc8,\xa4\xf0^,\xbc;\xef@\x04kA\xb3(\x16b\x97 3("\xc9\x14\'\x13\xc5\x9c\x9cq\x8bmg\xdcj\x93\x17\xf3<\xedw\x8a\xf7\x83\x1f\xe7\x9c\xfa\xd1\xd1\xd1\xc6\xc6F\r\xa6\xe4\xf1\xf1\xf1\xfe\xfe~~{{\xbb\xd5jM{\x0c\xb3\xeb\xe1\xe1\xa1\xddn\xcf\xbf\xf3D\xaf\xd7\xbb\xbd\xbdm4\x1aooo\xbb\xbb\xbb\x97\x97\x97;;;\xa3\xd1\xa8\xd3\xe9\xb4Z\xad\xaa\xaa...\x96\x96\x96\xca\xb2\xdc\xdf\xdf???_[[[^^\xbe\xb9\xb9\xd9\xdb\xdbk6\x9b\x13\xbb\t\x1f\xc0{!\xd6\xeb\xf5n\xb7\xfb\xf4\xf4\xb4\xb0\xb0p}}\xbd\xbe\xbe~rrR\x14\xc5\xc1\xc1AY\x96\x8dF\xe3\xea\xea\xaa\xaa\xaa\xc5\xc5\xc5V\xab\xd5\xe9tNOOWVV\x0e\x0f\x0f\x87\xc3\xe1\xc4.\xc0\xc70\xf7\xce\xd9`0(\xcb\xf2\xc7`\xf0\xbd\xdf\xffsk\xebgU\xfd\xf5\xe5K\xb3\xd9\xfc\xbb\xdd\xfecn\xee\xf9\xf9\xf9\xf5\xf5\xb5(\x8a\xe1pxww\xd7\xef\xf7\x8b\xa2X]]=;;\x9b\xd8z>\x8cz\xbb\xdd\xfe\xed?\xe2\xb7o\xb5\xaf_\x7f}\xf4\xe9S\xed\xf3\xe7\xffo\x16\xb3\xe3\xbf\xff\x11k\x9b\x9b\xb5\xcd\xcdI\xeda\xa6\xbd\xf7i\x86\x89\x11"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!\x12A\x88D\x10"\x11\x84H\x04!2e\xe3\xf1x<\x1e\xcfO{\x06\xb3\xee\xf8\xf8\xb8\xd7\xeby#2e///UU\t\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\x08\x91\x08B$\x82\x10\x89 D"\xccw\xbb\xdd\xd1h4\xed\x19\xcc\xae\x7f\xf3\xfb\x07q8\x9emk8\x97\xda\x00\x00\x00\x00IEND\xaeB`\x82'), + 'mime_type': 'application/vnd.oasis.opendocument.text'} + +class CommonTest(unittest.TestCase): + + def setUp(self): + bus = dbus.SessionBus() + proxy = bus.get_object(DS_DBUS_SERVICE, DS_DBUS_PATH) + self._data_store = dbus.Interface(proxy, DS_DBUS_INTERFACE) + + def create(self): + file_path = self._prepare_file() + + t = time.time() + uid = self._data_store.create(PROPS_WITHOUT_PREVIEW, file_path, True) + t = time.time() - t + return t, uid + + def update(self, uid): + file_path = self._prepare_file() + t = time.time() + self._data_store.update(uid, PROPS_WITH_PREVIEW, file_path, True) + t = time.time() - t + return t + + def find(self): + query = {'order_by': ['-mtime'], + 'limit': 80} + t = time.time() + results, count = self._data_store.find(query, []) + t = time.time() - t + return t + + def _prepare_file(self): + file_path = os.path.join(os.getcwd(), 'tests/funkyabi.odt') + f, tmp_path = tempfile.mkstemp() + os.close(f) + shutil.copyfile(file_path, tmp_path) + return tmp_path + +class FunctionalityTest(CommonTest): + + def testcreation(self): + t, uid = self.create() + assert uid + + def testupdate(self): + t, uid = self.create() + t = self.update(uid) + + def testresume(self): + t, uid = self.create() + props = self._data_store.get_properties(uid, byte_arrays=True) + #del props['uid'] + assert props == PROPS_WITHOUT_PREVIEW + + t = self.update(uid) + props = self._data_store.get_properties(uid, byte_arrays=True) + #del props['uid'] + assert props == PROPS_WITH_PREVIEW + + file_name = self._data_store.get_filename(uid) + + assert os.path.exists(file_name) + f = open(file_name, 'r') + f.close() + + """ + def testcustomproperties(self): + t, uid = self.create() + props = self._data_store.get_properties(uid) + props['custom_property'] = 'test' + self._data_store.update(uid, props, '', True) + + props = self._data_store.get_properties(uid) + assert props['custom_property'] == 'test' + + results, count = self._data_store.find({'custom_property': 'test'}, []) + assert count > 1 + for entry in results: + assert entry['custom_property'] == 'test' + uid = entry['uid'] + props = self._data_store.get_properties(uid) + assert props['custom_property'] == 'test' + """ + + def testfind(self): + t = self.find() + +class PerformanceTest(CommonTest): + + def _avg(self, l): + total = 0 + for i in l: + total += i + return total / len(l) + + def _test_perf(self, label, function, iterations): + t_max = 0 + t_min = sys.maxint + times = [] + for i in range(1, iterations): + t = function() + t_max = max(t, t_max) + t_min = min(t, t_min) + times.append(t) + + print '%s max: %.3fms min: %.3fms avg: %.3fms' % \ + (label, t_max * 1000, t_min * 1000, self._avg(times) * 1000) + + def testperformance(self): + iterations = 100 + + self._test_perf('Create', lambda: self.create()[0], iterations) + + t, uid = self.create() + self._test_perf('Update', lambda: self.update(uid), iterations) + + self._test_perf('Find', lambda: self.find(), iterations) + + +if __name__ == '__main__': + suite = unittest.TestSuite() + #suite.addTest(unittest.makeSuite(FunctionalityTest)) + suite.addTest(unittest.makeSuite(PerformanceTest)) + unittest.TextTestRunner().run(suite) + -- cgit v0.9.1