From 028e410af82b407a48f6c6b676d026f6c24077f9 Mon Sep 17 00:00:00 2001 From: Sascha Silbe Date: Sun, 22 May 2011 22:33:55 +0000 Subject: start implementing native API --- diff --git a/bin/gdatastore-service b/bin/gdatastore-service index d065f3a..8899fe0 100755 --- a/bin/gdatastore-service +++ b/bin/gdatastore-service @@ -56,6 +56,7 @@ def main(): bus = dbus.SessionBus() internal_api = datastore.InternalApi(BASE_DIR) dbus_api_sugar_v2 = datastore.DBusApiSugarV2(internal_api) + dbus_api_native_v1 = datastore.DBusApiNativeV1(internal_api) mainloop = gobject.MainLoop() bus.set_exit_on_disconnect(False) diff --git a/gdatastore/datastore.py b/gdatastore/datastore.py index d1457aa..2378abc 100644 --- a/gdatastore/datastore.py +++ b/gdatastore/datastore.py @@ -63,6 +63,157 @@ class GitError(DataStoreError): return self.__unicode__() +class DBusApiNativeV1(dbus.service.Object): + """Native gdatastore D-Bus API + """ + + def __init__(self, internal_api): + self._internal_api = internal_api + bus_name = dbus.service.BusName(DBUS_SERVICE_NATIVE_V1, + bus=dbus.SessionBus(), + replace_existing=False, + allow_replacement=False, + do_not_queue=True) + dbus.service.Object.__init__(self, bus_name, DBUS_PATH_NATIVE_V1) + self._internal_api.add_callback('change_metadata', + self.__change_metadata_cb) + self._internal_api.add_callback('delete', self.__delete_cb) + self._internal_api.add_callback('save', self.__save_cb) + + @dbus.service.signal(DBUS_INTERFACE_NATIVE_V1, signature='sssa{sv}') + def AddedNewVersion(self, tree_id, child_id, parent_id, metadata): + # pylint: disable-msg=C0103 + pass + + @dbus.service.signal(DBUS_INTERFACE_NATIVE_V1, signature='ssa{sv}') + def Created(self, tree_id, child_id, metadata): + # pylint: disable-msg=C0103 + pass + + @dbus.service.signal(DBUS_INTERFACE_NATIVE_V1, signature='ssa{sv}') + def ChangedMetadata(self, tree_id, version_id, metadata): + # pylint: disable-msg=C0103 + pass + + @dbus.service.signal(DBUS_INTERFACE_NATIVE_V1, signature='ss') + def Deleted(self, tree_id, version_id): + # pylint: disable-msg=C0103 + pass + + @dbus.service.method(DBUS_INTERFACE_NATIVE_V1, + in_signature='a{sv}s', out_signature='ss', + async_callbacks=('async_cb', 'async_err_cb'), + byte_arrays=True) + def create(self, metadata, data_path, async_cb, async_err_cb): + """ + - add new entry, assign ids + - data='' indicates no data to store + - bad design? (data OOB) + """ + # TODO: what about transfer_ownership/delete_after? + self._internal_api.save(tree_id='', parent_id='', metadata=metadata, + path=data_path, delete_after=True, + async_cb=async_cb, + async_err_cb=async_err_cb) + + @dbus.service.method(DBUS_INTERFACE_NATIVE_V1, + in_signature='ssa{sv}s', out_signature='s', + async_callbacks=('async_cb', 'async_err_cb'), + byte_arrays=True) + def add_version(self, tree_id, parent_id, metadata, data_path, async_cb, + async_err_cb): + """ + - add new version to existing object + """ + def success_cb(tree_id, child_id): + async_cb(child_id) + + if not tree_id: + raise ValueError('No tree_id given') + + if not parent_id: + raise ValueError('No parent_id given') + + self._internal_api.save(tree_id, parent_id, metadata, data_path, + delete_after=True, + async_cb=success_cb, + async_err_cb=async_err_cb) + + @dbus.service.method(DBUS_INTERFACE_NATIVE_V1, + in_signature='ssa{sv}', out_signature='', + byte_arrays=True) + def change_metadata(self, tree_id, version_id, metadata): + """ + - change the metadata of an existing version + """ + object_id = (tree_id, version_id) + self._internal_api.change_metadata(object_id, metadata) + + @dbus.service.method(DBUS_INTERFACE_NATIVE_V1, + in_signature='ss', out_signature='') + def delete(self, tree_id, version_id): + object_id = (tree_id, version_id) + self._internal_api.delete(object_id) + + @dbus.service.method(DBUS_INTERFACE_NATIVE_V1, + in_signature='a{sv}a{sv}', out_signature='aa{sv}u', + byte_arrays=True) + def find(self, query_dict, options): + return self._internal_api.find(query_dict, options) + + @dbus.service.method(DBUS_INTERFACE_NATIVE_V1, + in_signature='ss', out_signature='s', + sender_keyword='sender') + def get_data_path(self, tree_id, version_id, sender=None): + object_id = (tree_id, version_id) + return self._internal_api.get_data_path(object_id, sender=sender) + + @dbus.service.method(DBUS_INTERFACE_NATIVE_V1, + in_signature='ss', out_signature='a{sv}') + def get_metadata(self, tree_id, version_id): + object_id = (tree_id, version_id) + return self._internal_api.get_properties(object_id) + + @dbus.service.method(DBUS_INTERFACE_NATIVE_V1, + in_signature='a{sv}sa{sv}', out_signature='aa{sv}u', + byte_arrays=True) + def text_search(self, query_dict, query_string, options): + return self._internal_api.find(query_dict, options, query_string) + + @dbus.service.method(DBUS_INTERFACE_NATIVE_V1, + in_signature='sssa{sv}s', out_signature='', + async_callbacks=('async_cb', 'async_err_cb'), + byte_arrays=True) + def restore(self, tree_id, parent_id, version_id, metadata, data_path, + async_cb, async_err_cb): + """ + - add a new version with the given ids + - there must be no existing entry with the same (tree_id, version_id) + - if parent_id != '' there must be an existing entry (tree_id, parent_id) + - if parent_id = '', there must be no existing entry with the same tree_id and no parent_id + """ + if not tree_id: + raise ValueError('No tree_id given') + + metadata['version_id'] = version_id + self._internal_api.save(tree_id, parent_id, metadata, data_path, + delete_after=True, + async_cb=async_cb, + async_err_cb=async_err_cb) + + def __change_metadata_cb(self, (tree_id, version_id), metadata): + self.ChangedMetadata(tree_id, version_id, metadata) + + def __delete_cb(self, (tree_id, version_id)): + self.Deleted(tree_id, version_id) + + def __save_cb(self, tree_id, child_id, parent_id, metadata): + if parent_id: + self.AddedNewVersion(tree_id, child_id, parent_id, metadata) + else: + self.Created(tree_id, child_id, metadata) + + class DBusApiSugarV2(dbus.service.Object): """Compatibility layer for the Sugar 0.84+ data store D-Bus API """ @@ -320,6 +471,9 @@ class InternalApi(object): return self._index.find_unique_values(name) + def get_properties(self, object_id): + return self._index.retrieve(object_id) + def save(self, tree_id, parent_id, metadata, path, delete_after, async_cb, async_err_cb): logging.debug('save(%r, %r, %r, %r, %r)', tree_id, parent_id, diff --git a/org.silbe.GDataStore.service.in b/org.silbe.GDataStore.service.in new file mode 100644 index 0000000..9cee91b --- /dev/null +++ b/org.silbe.GDataStore.service.in @@ -0,0 +1,3 @@ +[D-BUS Service] +Name = org.silbe.GDataStore +Exec = %(install_dir)s/bin/gdatastore-service diff --git a/tests/native_api_v1.txt b/tests/native_api_v1.txt new file mode 100644 index 0000000..5df64d8 --- /dev/null +++ b/tests/native_api_v1.txt @@ -0,0 +1,143 @@ +>>> import os +>>> import tempfile +>>> import time + +Define some helper functions +>>> def test_unique(items): +... return not [True for e in items if items.count(e) > 1] +>>> def to_native(value): +... if isinstance(value, list): +... return [to_native(e) for e in value] +... elif isinstance(value, dict): +... return dict([(to_native(k), to_native(v)) for k, v in value.items()]) +... elif isinstance(value, unicode): +... return unicode(value) +... elif isinstance(value, str): +... return str(value) +... return value +>>> def create_temp_file(content): +... fd, file_name = tempfile.mkstemp() +... f = os.fdopen(fd, 'w') +... f.write(content) +... f.close() +... return file_name + + +Connect to data store using DBus: +>>> import dbus +>>> DS_DBUS_SERVICE = 'org.silbe.GDataStore' +>>> DS_DBUS_INTERFACE = 'org.silbe.GDataStore1' +>>> DS_DBUS_PATH = '/org/silbe/GDataStore1' +>>> bus = dbus.SessionBus() +>>> ds = dbus.Interface(bus.get_object(DS_DBUS_SERVICE, DS_DBUS_PATH), DS_DBUS_INTERFACE) + + +Make sure we're starting from an empty datastore: +>>> assert ds.find({}, {}) == ([], 0) + + +Create something to play with: +>>> o1_oid = ds.create({'title': 'DS test object 1', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest1'}, '') +>>> assert isinstance(o1_oid, tuple) and len(o1_oid) == 2 and isinstance(o1_oid[0], basestring) and isinstance(o1_oid[1], basestring) +>>> o2_oid = ds.create({'title': 'DS test object 2', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest2'}, '') +>>> assert isinstance(o2_oid, tuple) and len(o2_oid) == 2 and isinstance(o2_oid[0], basestring) and isinstance(o2_oid[1], basestring) +>>> o3_oid = ds.create({'title': 'DS test object 3', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest2'}, '') +>>> assert isinstance(o3_oid, tuple) and len(o3_oid) == 2 and isinstance(o3_oid[0], basestring) and isinstance(o3_oid[1], basestring) +>>> assert test_unique([o1_oid, o2_oid, o3_oid]) + + +Check everything is there: +>>> assert sorted(to_native(ds.find({}, {'metadata': ['title', 'activity']}, byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 1', u'activity': 'org.sugarlabs.DataStoreTest1'}, +... {u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest2'}, +... {u'title': 'DS test object 3', u'activity': 'org.sugarlabs.DataStoreTest2'}] +>>> ds.get_data_path(*o1_oid, byte_arrays=True) +dbus.String(u'') +>>> ds.get_data_path(*o2_oid, byte_arrays=True) +dbus.String(u'') +>>> ds.get_data_path(*o3_oid, byte_arrays=True) +dbus.String(u'') + + +Change some metadata without creating a new version: +>>> ds.change_metadata(o1_oid[0], o1_oid[1], {'title': 'DS test object 1 updated', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest1', 'tags': 'foo'}) +>>> ds.change_metadata(o2_oid[0], o2_oid[1], {'title': 'DS test object 2', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest1', 'tags': 'bar baz'}) +>>> ds.change_metadata(o3_oid[0], o3_oid[1], {'title': 'DS test object 2', 'mime_type': 'text/html', 'activity': 'org.sugarlabs.DataStoreTest3', 'timestamp': 10000}) +>>> assert sorted(to_native(ds.find({}, {'metadata': ['title', 'activity']}, byte_arrays=True)[0])) == \ +... [{u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 1 updated'}, +... {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 2'}, +... {u'activity': 'org.sugarlabs.DataStoreTest3', u'title': 'DS test object 2'}] +>>> unicode(ds.get_metadata(*o1_oid)['title']) +u'DS test object 1 updated' + + +Retrieve metadata for a single entry, ignoring variable data: +>>> d=dict(ds.get_metadata(*o3_oid, byte_arrays=True)) +>>> del d['tree_id'], d['version_id'], d['timestamp'], d['creation_time'] +>>> assert to_native(d) == {u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'} + + +Find entries using "standard" metadata: +>>> assert sorted(to_native(ds.find({'mime_type': ['text/plain']}, {'metadata': ['title', 'activity', 'mime_type', 'tags']}, byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest1', u'mime_type': 'text/plain', u'tags': 'bar baz'}, +... {u'title': 'DS test object 1 updated', u'activity': 'org.sugarlabs.DataStoreTest1', u'mime_type': 'text/plain', u'tags': 'foo'}] +>>> assert sorted(to_native(ds.find({'mime_type': ['text/html']}, {'metadata': ['title', 'activity', 'mime_type', 'tags']}, byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'}] +>>> assert sorted(to_native(ds.find({'tree_id': o3_oid[0]}, {'metadata': ['title', 'activity', 'mime_type']}, byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'}] +>>> assert sorted(to_native(ds.find({'timestamp': (9000, 11000)}, {'metadata': ['title', 'activity', 'mime_type']}, byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'}] + +Find entries using "non-standard" metadata (only works with dict-based queries or prefixed Xapian query strings): +>>> assert sorted(to_native(ds.find({'title': 'DS test object 2'}, {'metadata': ['title', 'activity', 'mime_type', 'tags']}, byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'}, +... {u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest1', u'mime_type': 'text/plain', u'tags': 'bar baz'}] + + +You can specify a (primary) sort order. Please note that the secondary sort order is undefined / implementation-dependent. +>>> assert to_native(ds.find({}, {'metadata': ['title', 'activity'], 'order_by': ['+title']}, byte_arrays=True)[0]) == \ +... [{u'activity': 'org.sugarlabs.DataStoreTest3', u'title': 'DS test object 2'}, +... {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 2'}, +... {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 1 updated'}] +>>> assert to_native(ds.find({}, {'metadata': ['title', 'activity'], 'order_by': ['-title']}, byte_arrays=True)[0]) == \ +... [{u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 1 updated'}, +... {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 2'}, +... {u'activity': 'org.sugarlabs.DataStoreTest3', u'title': 'DS test object 2'}] + + +Delete an entry: +>>> ds.delete(*o1_oid) +>>> assert sorted(to_native(ds.find({}, {'metadata': ['title', 'activity']}, byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest1'}, +... {u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest3'}] + + +Create an entry with content: +>>> dog_content = 'The quick brown dog jumped over the lazy fox.' +>>> dog_props = {'title': 'dog/fox story', 'mime_type': 'text/plain'} +>>> dog_file_name = create_temp_file(dog_content) +>>> dog_oid = ds.create(dog_props, dog_file_name) + + +Retrieve and verify the entry with content: +>>> dog_retrieved = ds.get_data_path(*dog_oid) +>>> assert(file(dog_retrieved).read() == dog_content) +>>> os.remove(dog_retrieved) + + +Update the entry content (creates a new version): +>>> dog_content_2 = 'The quick brown fox jumped over the lazy dog.' +>>> dog_file_name = create_temp_file(dog_content_2) +>>> dog_updated_version_id = ds.add_version(dog_oid[0], dog_oid[1], dog_props, dog_file_name) + + +Verify updated content: +>>> dog_retrieved = ds.get_data_path(dog_oid[0], dog_updated_version_id) +>>> assert(file(dog_retrieved).read() == dog_content_2) +>>> os.remove(dog_retrieved) + + +Verify old content is still accessible: +>>> dog_retrieved = ds.get_data_path(*dog_oid) +>>> assert(file(dog_retrieved).read() == dog_content) +>>> os.remove(dog_retrieved) diff --git a/tests/runalltests.py b/tests/runalltests.py index 26df032..1ed42c9 100755 --- a/tests/runalltests.py +++ b/tests/runalltests.py @@ -32,6 +32,7 @@ logging.basicConfig(level=logging.WARN, DOCTESTS = [ + 'native_api_v1.txt', 'sugar_api_v2.txt', ] DOCTEST_OPTIONS = (doctest.ELLIPSIS | doctest.REPORT_ONLY_FIRST_FAILURE | @@ -47,6 +48,18 @@ ENVIRONMENT_WHITELIST = [ 'GDATASTORE_LOGLEVEL', ] + +def write_service_file(service_dir, executable, bus_name): + service_path = os.path.join(service_dir, bus_name + '.service') + service_file = file(service_path, 'w') + service_file.write(""" + [D-BUS Service] + Name = %s + Exec = %s + """.replace(' ', '') % (bus_name, executable)) + service_file.close() + + def setup(): """Prepare for testing and return environment. @@ -69,19 +82,14 @@ def setup(): python_path = [basedir] + python_path environment['PYTHONPATH'] = ':'.join(python_path) environment['PATH'] = os.path.join(basedir, 'bin')+':'+os.environ['PATH'] - - servicedir = os.path.join(environment['HOME'], 'dbus-1', 'services') - servicepath = os.path.join(servicedir, 'org.laptop.sugar.DataStore.service') - os.makedirs(servicedir) - servicefile = file(servicepath, 'w') - servicefile.write(""" - [D-BUS Service] - Name = org.laptop.sugar.DataStore - Exec = %s/bin/gdatastore-service - """.replace(' ', '') % (basedir, )) - servicefile.close() environment['XDG_DATA_DIRS'] = environment['HOME'] + service_dir = os.path.join(environment['HOME'], 'dbus-1', 'services') + os.makedirs(service_dir) + executable = os.path.join(basedir, 'bin', 'gdatastore-service') + write_service_file(service_dir, executable, 'org.laptop.sugar.DataStore') + write_service_file(service_dir, executable, 'org.silbe.GDataStore') + os.setpgid(0, 0) # prevent suicide in cleanup() signal.signal(signal.SIGTERM, signal.SIG_IGN) -- cgit v0.9.1