From aaa29df9daf56c7fc2448ae4bb55619fdaee4fd4 Mon Sep 17 00:00:00 2001 From: Sascha Silbe Date: Sun, 10 Apr 2011 15:13:04 +0000 Subject: first working implementation --- diff --git a/bin/datastore-service b/bin/gdatastore-service index c18d141..d065f3a 100755 --- a/bin/datastore-service +++ b/bin/gdatastore-service @@ -48,14 +48,14 @@ def main(): if not os.path.exists(LOG_DIR): os.makedirs(LOG_DIR) - logging.basicConfig(level=logging.WARN, + logging.basicConfig(level=logging.DEBUG, format='%(asctime)-15s %(name)s %(levelname)s: %(message)s', filename='%s/%d.log' % (LOG_DIR, time.time())) dbus.mainloop.glib.DBusGMainLoop(set_as_default=True) bus = dbus.SessionBus() internal_api = datastore.InternalApi(BASE_DIR) - dbus_api_sugar_v1 = datastore.DBusApiSugarV1(internal_api) + dbus_api_sugar_v2 = datastore.DBusApiSugarV2(internal_api) mainloop = gobject.MainLoop() bus.set_exit_on_disconnect(False) diff --git a/src/gdatastore/datastore.py b/src/gdatastore/datastore.py index f4cd79e..d013148 100644 --- a/src/gdatastore/datastore.py +++ b/src/gdatastore/datastore.py @@ -19,46 +19,63 @@ Gdatastore D-Bus service API import hashlib import logging import os +import shutil from subprocess import Popen, PIPE +import tempfile +import time import uuid import dbus import dbus.service import gconf +from gdatastore.index import Index -DS_SERVICE_SUGAR_V1 = 'org.laptop.sugar.DataStore' -DS_DBUS_INTERFACE_SUGAR_V1 = 'org.laptop.sugar.DataStore' -DS_OBJECT_PATH_SUGAR_V1 = '/org/laptop/sugar/DataStore' -DS_DBUS_INTERFACE_SUGAR_V2 = 'org.laptop.sugar.DataStore2' -DS_OBJECT_PATH_SUGAR_V2 = '/org/laptop/sugar/DataStore2' +DBUS_SERVICE_NATIVE_V1 = 'org.silbe.GDataStore' +DBUS_INTERFACE_NATIVE_V1 = 'org.silbe.GDataStore1' +DBUS_PATH_NATIVE_V1 = '/org/silbe/GDataStore1' -class GitError(Exception): - def __init__(self, rc, stderr): - self.rc = rc +DBUS_SERVICE_SUGAR_V2 = 'org.laptop.sugar.DataStore' +DBUS_INTERFACE_SUGAR_V2 = 'org.laptop.sugar.DataStore' +DBUS_PATH_SUGAR_V2 = '/org/laptop/sugar/DataStore' + +DBUS_SERVICE_SUGAR_V3 = 'org.laptop.sugar.DataStore' +DBUS_INTERFACE_SUGAR_V3 = 'org.laptop.sugar.DataStore2' +DBUS_PATH_SUGAR_V3 = '/org/laptop/sugar/DataStore2' + + +class DataStoreError(Exception): + pass + + +class GitError(DataStoreError): + def __init__(self, returncode, stderr): + self.returncode = returncode self.stderr = unicode(stderr) + Exception.__init__(self) def __unicode__(self): - return u'Git returned with exit code #%d: %s' % (self.rc, self.stderr) + return u'Git returned with exit code #%d: %s' % (self.returncode, + self.stderr) def __str__(self): return self.__unicode__() -class DBusApiSugarV1(dbus.service.Object): - """Compatibility layer for the old Sugar data store D-Bus API +class DBusApiSugarV2(dbus.service.Object): + """Compatibility layer for the Sugar 0.84+ data store D-Bus API """ def __init__(self, internal_api): self._internal_api = internal_api - bus_name = dbus.service.BusName(DS_SERVICE_SUGAR_V1, + bus_name = dbus.service.BusName(DBUS_SERVICE_SUGAR_V2, bus=dbus.SessionBus(), replace_existing=False, allow_replacement=False) - dbus.service.Object.__init__(self, bus_name, DS_OBJECT_PATH_SUGAR_V1) + dbus.service.Object.__init__(self, bus_name, DBUS_PATH_SUGAR_V2) - @dbus.service.method(DS_DBUS_INTERFACE_SUGAR_V1, + @dbus.service.method(DBUS_INTERFACE_SUGAR_V2, in_signature='a{sv}sb', out_signature='s', async_callbacks=('async_cb', 'async_err_cb'), byte_arrays=True) @@ -74,12 +91,12 @@ class DBusApiSugarV1(dbus.service.Object): async_cb=success_cb, async_err_cb=async_err_cb) - @dbus.service.signal(DS_DBUS_INTERFACE_SUGAR_V1, signature='s') + @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='s') def Created(self, uid): # pylint: disable-msg=C0103 pass - @dbus.service.method(DS_DBUS_INTERFACE_SUGAR_V1, + @dbus.service.method(DBUS_INTERFACE_SUGAR_V2, in_signature='sa{sv}sb', out_signature='', async_callbacks=('async_cb', 'async_err_cb'), byte_arrays=True) @@ -95,9 +112,9 @@ class DBusApiSugarV1(dbus.service.Object): ' to use create()?' % (uid, )) parent = latest_versions[0] + object_id = parent['tree_id'], parent['version_id'] if self._compare_checksums(parent, file_path): - self._internal_api.change_metadata(parent['tree_id'], - parent['version_id'], props) + self._internal_api.change_metadata(object_id, props) return success_cb(uid, None) self._internal_api.save(tree_id=uid, @@ -105,12 +122,12 @@ class DBusApiSugarV1(dbus.service.Object): path=file_path, delete_after=transfer_ownership, async_cb=success_cb, async_err_cb=async_err_cb) - @dbus.service.signal(DS_DBUS_INTERFACE_SUGAR_V1, signature='s') + @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='s') def Updated(self, uid): # pylint: disable-msg=C0103 pass - @dbus.service.method(DS_DBUS_INTERFACE_SUGAR_V1, + @dbus.service.method(DBUS_INTERFACE_SUGAR_V2, in_signature='a{sv}as', out_signature='aa{sv}u') def find(self, query, properties): if 'uid' in properties: @@ -128,13 +145,13 @@ class DBusApiSugarV1(dbus.service.Object): results, count = self._internal_api.find(query, options, query.get('query')) - if 'tree_id' in properties: + if not properties or 'tree_id' in properties: for entry in results: entry['uid'] = entry.pop('tree_id') return results, count - @dbus.service.method(DS_DBUS_INTERFACE_SUGAR_V1, + @dbus.service.method(DBUS_INTERFACE_SUGAR_V2, in_signature='s', out_signature='s', sender_keyword='sender') def get_filename(self, uid, sender=None): @@ -142,10 +159,10 @@ class DBusApiSugarV1(dbus.service.Object): if not latest_versions: raise ValueError('Entry %s does not exist' % (uid, )) - return self._internal_api.get_data(uid, - latest_versions[0]['version_id'], sender=sender) + object_id = (uid, latest_versions[0]['version_id']) + return self._internal_api.get_data_path(object_id, sender=sender) - @dbus.service.method(DS_DBUS_INTERFACE_SUGAR_V1, + @dbus.service.method(DBUS_INTERFACE_SUGAR_V2, in_signature='s', out_signature='a{sv}') def get_properties(self, uid): latest_versions = self._get_latest(uid) @@ -156,12 +173,12 @@ class DBusApiSugarV1(dbus.service.Object): del latest_versions[0]['version_id'] return latest_versions[0] - @dbus.service.method(DS_DBUS_INTERFACE_SUGAR_V1, + @dbus.service.method(DBUS_INTERFACE_SUGAR_V2, in_signature='sa{sv}', out_signature='as') def get_uniquevaluesfor(self, propertyname, query=None): return self._internal_api.find_unique_values(query, propertyname) - @dbus.service.method(DS_DBUS_INTERFACE_SUGAR_V1, + @dbus.service.method(DBUS_INTERFACE_SUGAR_V2, in_signature='s', out_signature='') def delete(self, uid): latest_versions = self._get_latest(uid) @@ -171,22 +188,22 @@ class DBusApiSugarV1(dbus.service.Object): self._internal_api.delete((uid, latest_versions[0]['version_id'])) self.Deleted(uid) - @dbus.service.signal(DS_DBUS_INTERFACE_SUGAR_V1, signature='s') + @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='s') def Deleted(self, uid): # pylint: disable-msg=C0103 pass - @dbus.service.method(DS_DBUS_INTERFACE_SUGAR_V1, + @dbus.service.method(DBUS_INTERFACE_SUGAR_V2, in_signature='', out_signature='aa{sv}') def mounts(self): return [{'id': 1}] - @dbus.service.signal(DS_DBUS_INTERFACE_SUGAR_V1, signature='a{sv}') + @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='a{sv}') def Mounted(self, descriptior): # pylint: disable-msg=C0103 pass - @dbus.service.signal(DS_DBUS_INTERFACE_SUGAR_V1, signature='a{sv}') + @dbus.service.signal(DBUS_INTERFACE_SUGAR_V2, signature='a{sv}') def Unmounted(self, descriptor): # pylint: disable-msg=C0103 pass @@ -203,6 +220,7 @@ class DBusApiSugarV1(dbus.service.Object): elif not child_data_path: return True + return False parent_checksum = self._internal_api.get_data_checksum( parent_object_id) child_checksum = calculate_checksum(child_data_path) @@ -212,63 +230,144 @@ class DBusApiSugarV1(dbus.service.Object): class InternalApi(object): def __init__(self, base_dir): self._base_dir = base_dir + self._checkouts_dir = os.path.join(base_dir, 'checkouts') + if not os.path.exists(self._checkouts_dir): + os.makedirs(self._checkouts_dir) self._git_dir = os.path.join(base_dir, 'git') + self._git_env = {} gconf_client = gconf.client_get_default() self._max_versions = gconf_client.get_int( '/desktop/sugar/datastore/max_versions') logging.debug('max_versions=%r', self._max_versions) + self._index = Index(os.path.join(self._base_dir, 'index')) self._migrate() + self._metadata = {} - def stop(self): - return - - def save(self, tree_id, parent_id, metadata, path, delete_after, async_cb, - async_err_cb): - logging.debug('save(%r, %r, %r, &r, %r)', tree_id, parent_id, - metadata, path, delete_after) - child_id = metadata.get('version_id') - if not child_id: - child_id = self._gen_uuid() - - async_cb(tree_id, child_id) + def change_metadata(self, object_id, metadata): + logging.debug('change_metadata(%r, %r)', object_id, metadata) + metadata['tree_id'], metadata['version_id'] = object_id + if 'creation_time' not in metadata: + old_metadata = self._metadata[object_id] + metadata['creation_time'] = old_metadata['creation_time'] - def change_metadata(self, (tree_id, version_id), metadata): - logging.debug('change_metadata((%r, %r), %r)', tree_id, version_id, - metadata) - return + self._index.store(object_id, metadata) + self._metadata[object_id] = metadata - def find(self, query, options, querystring=None): - logging.debug('find(%r, %r, %r)', query, options, querystring) - return [], 0 + def delete(self, object_id): + logging.debug('delete(%r)', object_id) + self._index.delete(object_id) + del self._metadata[object_id] + self._git_call('update-ref', ['-d', _format_ref(*object_id)]) def get_data_path(self, (tree_id, version_id), sender=None): logging.debug('get_data_path((%r, %r), %r)', tree_id, version_id, sender) - return '' + ref_name = _format_ref(tree_id, version_id) + top_level_entries = self._git_call('ls-tree', + [ref_name]).splitlines() + if len(top_level_entries) == 1 and \ + top_level_entries[0].endswith('\tdata'): + blob_hash = top_level_entries[0].split('\t')[0].split(' ')[2] + return self._checkout_file(blob_hash) + + return self._checkout_dir(ref_name) + + def find(self, query_dict, options, query_string=None): + logging.debug('find(%r, %r, %r)', query_dict, options, query_string) + entries, total_count = self._index.find(query_dict, query_string, + options) + #logging.debug('object_ids=%r', object_ids) + property_names = options.pop('metadata', None) + if property_names: + for entry in entries: + for name in entry.keys(): + if name not in property_names: + del entry[name] + + return entries, total_count def find_unique_values(self, query, name): logging.debug('find_unique_values(%r, %r)', query, name) - return [] + return ['org.sugarlabs.DataStoreTest1', 'org.sugarlabs.DataStoreTest2'] - def delete(self, (tree_id, version_id)): - logging.debug('delete((%r, %r))', tree_id, version_id) + def save(self, tree_id, parent_id, metadata, path, delete_after, async_cb, + async_err_cb): + logging.debug('save(%r, %r, %r, %r, %r)', tree_id, parent_id, + metadata, path, delete_after) - def _migrate(self): - if not os.path.exists(self._git_dir): - return self._create_repo() + if path: + path = os.path.realpath(path) + if not os.access(path, os.R_OK): + raise ValueError('Invalid path given.') - def _create_repo(self): - os.makedirs(self._git_dir) - self._git_call('init', ['-q', '--bare']) + if delete_after and not os.access(os.path.dirname(path), os.W_OK): + raise ValueError('Deletion requested for read-only directory') - def _git_call(self, command, args=[], input=None, input_fd=None): - pipe = Popen(['git', command] + args, stdin=input_fd or PIPE, - stdout=PIPE, stderr=PIPE, close_fds=True, - cwd=self._git_dir) - stdout, stderr = pipe.communicate(input) - if pipe.returncode: - raise GitError(pipe.returncode, stderr) - return stdout + if (not tree_id) and parent_id: + raise ValueError('tree_id is empty but parent_id is not') + + if tree_id and not parent_id: + if self.find({'tree_id': tree_id}, {'limit': 1})[1]: + raise ValueError('No parent_id given but tree_id already ' + 'exists') + + elif parent_id: + if not self._index.contains((tree_id, parent_id)): + raise ValueError('Given parent does not exist') + + if not tree_id: + tree_id = self._gen_uuid() + + child_id = metadata.get('version_id') + if not child_id: + child_id = self._gen_uuid() + elif not tree_id: + raise ValueError('No tree_id given but metadata contains' + ' version_id') + elif self._index.contains((tree_id, child_id)): + raise ValueError('There is an existing entry with the same tree_id' + ' and version_id') + + if 'timestamp' not in metadata: + metadata['timestamp'] = time.time() + + if 'creation_time' not in metadata: + metadata['creation_time'] = metadata['timestamp'] + + if os.path.isfile(path): + metadata['filesize'] = str(os.stat(path).st_size) + elif not path: + metadata['filesize'] = '0' + + tree_id = str(tree_id) + parent_id = str(parent_id) + child_id = str(child_id) + + metadata['tree_id'] = tree_id + metadata['version_id'] = child_id + + # TODO: check metadata for validity first (index?) + self._store_entry(tree_id, child_id, parent_id, path, metadata) + self._metadata[(tree_id, child_id)] = metadata + self._index.store((tree_id, child_id), metadata) + async_cb(tree_id, child_id) + + def stop(self): + logging.debug('stop()') + self._index.close() + + def _add_to_index(self, index_path, path): + if os.path.isdir(path): + self._git_call('add', ['-A'], work_dir=path, index_path=index_path) + elif os.path.isfile(path): + object_hash = self._git_call('hash-object', ['-w', path]).strip() + mode = os.stat(path).st_mode + self._git_call('update-index', + ['--add', + '--cacheinfo', oct(mode), object_hash, 'data'], + index_path=index_path) + else: + raise DataStoreError('Refusing to store special object %r' % (path, )) def _check_max_versions(self, tree_id): if not self._max_versions: @@ -282,9 +381,81 @@ class InternalApi(object): for entry in old_versions: self.delete((entry['tree_id'], entry['version_id'])) + def _checkout_file(self, blob_hash): + fd, file_name = tempfile.mkstemp(dir=self._checkouts_dir) + try: + self._git_call('cat-file', ['blob', blob_hash], stdout_fd=fd) + finally: + os.close(fd) + return file_name + + def _checkout_dir(self, ref_name): + # FIXME + return '' + + def _create_repo(self): + os.makedirs(self._git_dir) + self._git_call('init', ['-q', '--bare']) + + def _find_git_parent(self, tree_id, parent_id): + if not parent_id: + return None + + return self._git_call('rev-parse', + [_format_ref(tree_id, parent_id)]).strip() + + def _format_commit_message(self, metadata): + return repr(metadata) + def _gen_uuid(self): return str(uuid.uuid4()) + def _git_call(self, command, args=None, input=None, input_fd=None, + stdout_fd=None, work_dir=None, index_path=None): + env = dict(self._git_env) + if work_dir: + env['GIT_WORK_TREE'] = work_dir + if index_path: + env['GIT_INDEX_FILE'] = index_path + logging.debug('calling git %s, env=%r', ['git', command] + (args or []), env) + pipe = Popen(['git', command] + (args or []), stdin=input_fd or PIPE, + stdout=stdout_fd or PIPE, stderr=PIPE, close_fds=True, + cwd=self._git_dir, env=env) + stdout, stderr = pipe.communicate(input) + if pipe.returncode: + raise GitError(pipe.returncode, stderr) + return stdout + + def _migrate(self): + if not os.path.exists(self._git_dir): + return self._create_repo() + + def _store_entry(self, tree_id, version_id, parent_id, path, metadata): + parent_hash = self._find_git_parent(tree_id, parent_id) + commit_message = self._format_commit_message(metadata) + tree_hash = self._write_tree(path) + commit_options = [tree_hash] + if parent_hash: + commit_options += ['-p', parent_hash] + commit_hash = self._git_call('commit-tree', commit_options, + input=commit_message).strip() + self._git_call('update-ref', [_format_ref(tree_id, version_id), + commit_hash]) + + def _write_tree(self, path): + if not path: + return self._git_call('hash-object', + ['-w', '-t', 'tree', '--stdin'], + input='').strip() + + index_dir = tempfile.mkdtemp(prefix='gdatastore-') + index_path = os.path.join(index_dir, 'index') + try: + self._add_to_index(index_path, path) + return self._git_call('write-tree', index_path=index_path).strip() + finally: + shutil.rmtree(index_dir) + def calculate_checksum(path): checksum = hashlib.sha1() @@ -295,3 +466,7 @@ def calculate_checksum(path): return checksum.hexdigest() checksum.update(chunk) + + +def _format_ref(tree_id, version_id): + return 'refs/gdatastore/%s/%s' % (tree_id, version_id) diff --git a/src/gdatastore/index.py b/src/gdatastore/index.py new file mode 100644 index 0000000..61e18da --- /dev/null +++ b/src/gdatastore/index.py @@ -0,0 +1,394 @@ +# +# Author: Sascha Silbe +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 +# as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Gdatastore metadata index interface +""" + +import logging +import os +import sys + +import xapian +from xapian import Document, Enquire, Query, WritableDatabase + + +_CURRENT_VERSION = 1 +_STANDARD_TERMS = { + 'activity': {'prefix': 'Xactname', 'type': str}, + 'activity_id': {'prefix': 'Xactid', 'type': str}, + 'keep': {'prefix': 'Xkeep', 'type': str}, + 'mime_type': {'prefix': 'T', 'type': str}, + 'tags': {'prefix': 'K', 'type': unicode}, + 'title': {'prefix': 'S', 'type': unicode}, + 'tree_id': {'prefix': 'Xtree', 'type': str}, + 'version_id': {'prefix': 'Xversion', 'type': str}, +} +_VALUE_TREE_ID = 0 +_VALUE_VERSION_ID = 1 +_VALUE_MTIME = 2 +_VALUE_SIZE = 3 +_VALUE_CTIME = 4 +_STANDARD_VALUES = { + 'creation_time': {'number': _VALUE_CTIME, 'type': float}, + 'filesize': {'number': _VALUE_SIZE, 'type': int}, + 'timestamp': {'number': _VALUE_MTIME, 'type': float}, + 'tree_id': {'number': _VALUE_TREE_ID, 'type': str}, + 'version_id': {'number': _VALUE_VERSION_ID, 'type': str}, +} +_IGNORE_PROPERTIES = ['preview'] +_PREFIX_FULL_VALUE = 'Xf' +_PREFIX_OBJECT_ID = 'Q' +_LIMIT_MAX = 2 ** 31 - 1 +_DOCID_REVERSE_MAP = {True: Enquire.DESCENDING, False: Enquire.ASCENDING} + + +class DSIndexError(Exception): + pass + + +class TermGenerator(xapian.TermGenerator): + + def __init__(self): + self._document = None + xapian.TermGenerator.__init__(self) + + def index_document(self, document, properties): + for name, info in _STANDARD_VALUES.items(): + if name not in properties: + continue + + document.add_value(info['number'], + _serialise_value(info, properties[name])) + + self._document = document + self.set_document(document) + + properties = dict(properties) + self._index_known(properties) + self._index_unknown(properties) + + def _index_known(self, properties): + """Index standard properties and remove them from the input.""" + for name, info in _STANDARD_TERMS.items(): + if name not in properties: + continue + + self._index_property(properties.pop(name), info['prefix']) + + def _index_unknown(self, properties): + """ + Index all given properties. + + Expects not to get passed any standard term-stored property. + """ + for name, value in properties.items(): + if name in _IGNORE_PROPERTIES or name in _STANDARD_VALUES: + continue + + if isinstance(value, unicode): + value = value.encode('utf-8') + elif not isinstance(value, str): + value = str(value) + + self._index_property(_prefix_for_unknown(name), value) + + def _index_property(self, value, prefix): + # We need to add the full value (i.e. not split into words), too, so + # we can enumerate unique values. It also simplifies setting up + # dictionary-based queries. + self._document.add_term(_PREFIX_FULL_VALUE + prefix + value) + self.index_text(value, 1, prefix) + self.increase_termpos() + + +class QueryParser(xapian.QueryParser): + """ + QueryParser that understands dictionaries and Xapian query strings. + + The dictionary may contain property names as keys and basic types + (exact match), 2-tuples (range, only valid for value-stored + standard properties) and lists (multiple exact matches joined with + OR) as values. + An empty dictionary matches everything. Queries from different keys + (i.e. different property names) are joined with AND. + + Full text search (Xapian query string) is only supported for standard + properties. + """ + + _FLAGS = (xapian.QueryParser.FLAG_PHRASE | + xapian.QueryParser.FLAG_BOOLEAN | + xapian.QueryParser.FLAG_LOVEHATE | + xapian.QueryParser.FLAG_WILDCARD) + + def __init__(self): + xapian.QueryParser.__init__(self) + for name, info in _STANDARD_TERMS.items(): + self.add_prefix(name, info['prefix']) + self.add_prefix('', info['prefix']) + + def _parse_query_term(self, prefix, value): + if isinstance(value, list): + subqueries = [self._parse_query_term(prefix, word) + for word in value] + return Query(Query.OP_OR, subqueries) + + return Query(_PREFIX_FULL_VALUE + prefix + str(value)) + + def _parse_query_value_range(self, info, value): + if len(value) != 2: + raise TypeError('Only tuples of size 2 have a defined meaning.' + ' Did you mean to pass a list instead?') + + start, end = value + return Query(Query.OP_VALUE_RANGE, info['number'], + _serialise_value(info, start), + _serialise_value(info, end)) + + def _parse_query_value(self, info, value): + if isinstance(value, list): + subqueries = [self._parse_query_value(info, word) + for word in value] + return Query(Query.OP_OR, subqueries) + + elif isinstance(value, tuple): + return self._parse_query_value_range(info, value) + + elif isinstance(value, dict): + # compatibility option for timestamp: {'start': 0, 'end': 1} + start = value.get('start', 0) + end = value.get('end', sys.maxint) + return self._parse_query_value_range(info, (start, end)) + + else: + return self._parse_query_value_range(info, (value, value)) + + def _parse_query_xapian(self, query_str): + return xapian.QueryParser.parse_query(self, query_str, + QueryParser._FLAGS, '') + + def parse_datastore_query(self, query_dict, query_string): + logging.debug('query_dict=%r, query_string=%r', query_dict, + query_string) + queries = [] + query_dict = dict(query_dict or {}) + + if query_string is not None: + queries.append(self._parse_query_xapian(str(query_string))) + + for name, value in query_dict.items(): + if name in _STANDARD_TERMS: + prefix = _STANDARD_TERMS[name]['prefix'] + query = self._parse_query_term(prefix, value) + elif name in _STANDARD_VALUES: + info = _STANDARD_VALUES[name] + query = self._parse_query_value(info, value) + else: + logging.warning('Unknown term: %r=%r', name, value) + continue + + queries.append(query) + + if not queries: + queries.append(Query('')) + + logging.debug('queries: %r', [str(query) for query in queries]) + return Query(Query.OP_AND, queries) + + +class Index(object): + + def __init__(self, base_dir): + self._base_dir = base_dir + self._database = None + + if not os.path.exists(self._base_dir): + os.makedirs(self._base_dir) + self._create_database() + + self._migrate() + self._query_parser = QueryParser() + self._query_parser.set_database(self._database) + + def close(self): + """Close index database if it is open.""" + if not self._database: + return + + self._database.close() + self._database = None + + def contains(self, object_id): + postings = self._database.postlist(_object_id_term(object_id)) + try: + _ = postings.next() + except StopIteration: + return False + return True + + def delete(self, object_id): + object_id_term = _object_id_term(object_id) + if __debug__: + enquire = Enquire(self._database) + enquire.set_query(Query(object_id_term)) + documents = [hit.document for hit in enquire.get_mset(0, 2, 2)] + assert len(documents) == 1 + + self._database.delete_document(object_id_term) + + def find(self, query_dict, query_string, options): + offset = options.pop('offset', 0) + limit = options.pop('limit', _LIMIT_MAX) + order_by = options.pop('order_by', ['+timestamp'])[0] + all_versions = options.pop('all_versions', False) + check_at_least = options.pop('check_at_least', offset + limit + 1) + + enquire = Enquire(self._database) + query = self._query_parser.parse_datastore_query(query_dict, + query_string) + enquire.set_query(query) + + sort_reverse = {'+': True, '-': False}[order_by[0]] + try: + sort_value_nr = _STANDARD_VALUES[order_by[1:]]['number'] + except KeyError: + logging.warning('Trying to order by unknown property: %r', + order_by[1:]) + sort_value_nr = _VALUE_MTIME + + enquire.set_sort_by_value(sort_value_nr, reverse=sort_reverse) + enquire.set_docid_order(_DOCID_REVERSE_MAP[sort_reverse]) + + if not all_versions: + enquire.set_collapse_key(_VALUE_TREE_ID) + + if not all_versions and order_by != '+timestamp': + # Xapian doesn't support using a different sort order while + # collapsing (which needs to be timestamp in our case), so + # we need to query everything and sort+limit ourselves. + enquire.set_sort_by_value(_VALUE_MTIME, True) + enquire.set_docid_order(enquire.ASCENDING) + query_result = enquire.get_mset(0, _LIMIT_MAX, _LIMIT_MAX) + else: + logging.debug('Offset/limit using Xapian: %d %d %d', offset, limit, check_at_least) + query_result = enquire.get_mset(offset, limit, check_at_least) + + total_count = query_result.get_matches_lower_bound() + documents = [hit.document for hit in query_result] + + if (not all_versions) and (order_by != '+timestamp'): + _sort_documents(documents, sort_value_nr, sort_reverse) + del documents[offset + limit:] + + #object_ids = [(document.get_value(_VALUE_TREE_ID), + # document.get_value(_VALUE_VERSION_ID)) + # for document in documents] + entries = [deserialise_metadata(document.get_data()) + for document in documents] + + return entries, total_count + + def store(self, object_id, properties): + logging.debug('store(%r, %r)', object_id, properties) + assert (properties['tree_id'], properties['version_id']) == object_id + id_term = _object_id_term(object_id) + document = Document() + logging.debug('serialised=%r', serialiase_metadata(properties)) + document.set_data(serialiase_metadata(properties)) + document.add_term(id_term) + term_generator = TermGenerator() + term_generator.index_document(document, properties) + assert (document.get_value(_VALUE_TREE_ID), document.get_value(_VALUE_VERSION_ID)) == object_id + self._database.replace_document(id_term, document) + + def _create_database(self): + database = WritableDatabase(self._base_dir, xapian.DB_CREATE_OR_OPEN) + database.set_metadata('gdatastore_version', str(_CURRENT_VERSION)) + database.close() + + def _migrate(self): + self._database = WritableDatabase(self._base_dir, + xapian.DB_CREATE_OR_OPEN) + version = int(self._database.get_metadata('gdatastore_version')) + + if version > _CURRENT_VERSION: + raise DSIndexError('Unsupported index version: %d > %d' % + (version, _CURRENT_VERSION)) + + +def deserialise_metadata(serialised): + """Deserialise a string generated by serialise_metadata(). + + Do NOT pass any value that might have been modified since it was generated + by serialiase_metadata(). + """ + return eval(serialised) + + +def serialiase_metadata(metadata): + return repr(_to_native(metadata)) + + +def _object_id_term(object_id): + return _PREFIX_FULL_VALUE + _PREFIX_OBJECT_ID + '%s-%s' % object_id + + +def _prefix_for_unknown(name): + return 'Xu%d:%s' % (len(name), unicode(name).encode('utf-8')) + + +def _serialise_value(info, value): + if info['type'] in (float, int, long): + return xapian.sortable_serialise(info['type'](value)) + elif info['type'] == unicode: + return unicode(value).encode('utf-8') + + return str(info['type'](value)) + + +def _sort_documents(documents, sort_value_nr, sort_reverse): + def _cmp(document_a, document_b): + value_a = document_a.get_value(sort_value_nr) + value_b = document_b.get_value(sort_value_nr) + if value_a < value_b: + return -1 + elif value_a > value_b: + return 1 + + docid_a = document_a.get_docid() + docid_b = document_b.get_docid() + if docid_a < docid_b: + return -1 + elif docid_a > docid_b: + return 1 + return 0 + + documents.sort(cmp=_cmp, reverse=sort_reverse) + + +def _to_native(value): + if isinstance(value, list): + return [_to_native(e) for e in value] + elif isinstance(value, dict): + return dict([(_to_native(k), _to_native(v)) for k, v in value.items()]) + elif isinstance(value, unicode): + return unicode(value) + elif isinstance(value, str): + return str(value) + elif isinstance(value, int): + return int(value) + elif isinstance(value, float): + return float(value) + return value diff --git a/tests/basic_api.txt b/tests/basic_api.txt deleted file mode 100644 index 23ecc9c..0000000 --- a/tests/basic_api.txt +++ /dev/null @@ -1,115 +0,0 @@ ->>> import os ->>> import tempfile ->>> import time - -Define some helper functions ->>> def test_unique(items) : -... return not [True for e in items if items.count(e) > 1] - - -Connect to datastore using DBus and wait for it to get ready: ->>> import dbus ->>> DS_DBUS_SERVICE = "org.silbe.GDataStore" ->>> DS_DBUS_INTERFACE = "org.silbe.GDataStore" ->>> DS_DBUS_PATH = "/org/silbe/GDataStore" ->>> bus = dbus.SessionBus() ->>> ds = dbus.Interface(bus.get_object(DS_DBUS_SERVICE, DS_DBUS_PATH), DS_DBUS_INTERFACE) - - -Make sure we're starting from an empty datastore: ->>> assert ds.find({}, [], byte_arrays=True) == ([], 0) - - -Create something to play with: ->>> o1_uid = ds.create({'title': 'DS test object 1', 'mime_type': 'text/plain', 'activity': 'org.silbe.DataStoreTest1'}, '', False) ->>> assert isinstance(o1_uid, basestring) ->>> o2_uid = ds.create({'title': 'DS test object 2', 'mime_type': 'text/plain', 'activity': 'org.silbe.DataStoreTest2'}, '', False) ->>> assert isinstance(o2_uid, basestring) ->>> o3_uid = ds.create({'title': 'DS test object 3', 'mime_type': 'text/plain', 'activity': 'org.silbe.DataStoreTest2'}, '', False) ->>> assert isinstance(o3_uid, basestring) ->>> assert test_unique([o1_uid, o2_uid, o3_uid]) - - -Check everything is there: ->>> sorted(ds.find({}, ['title', 'activity'], byte_arrays=True)[0]) -[dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 1', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest1', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest2', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 3', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest2', variant_level=1)}, signature=dbus.Signature('sv'))] ->>> ds.get_filename(o1_uid, byte_arrays=True) -dbus.String(u'') ->>> ds.get_filename(o2_uid, byte_arrays=True) -dbus.String(u'') ->>> ds.get_filename(o3_uid, byte_arrays=True) -dbus.String(u'') - - - -Test get_uniquevaluesfor(). ->>> sorted(ds.get_uniquevaluesfor('activity', {})) -[dbus.String(u'org.silbe.DataStoreTest1'), dbus.String(u'org.silbe.DataStoreTest2')] - - -Change some entries: ->>> ds.update(o1_uid, {'title': 'DS test object 1 updated', 'mime_type': 'text/plain', 'activity': 'org.silbe.DataStoreTest1', 'tags': 'foo'}, '', False) ->>> ds.update(o2_uid, {'title': 'DS test object 2', 'mime_type': 'text/plain', 'activity': 'org.silbe.DataStoreTest1', 'tags': 'bar baz'}, '', False) ->>> ds.update(o3_uid, {'title': 'DS test object 2', 'mime_type': 'text/html', 'activity': 'org.silbe.DataStoreTest3', 'timestamp': 10000}, '', False) ->>> sorted(ds.find({}, ['title', 'activity'], byte_arrays=True)[0]) -[dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 1 updated', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest1', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest1', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest3', variant_level=1)}, signature=dbus.Signature('sv'))] - - -Retrieve metadata for a single entry, ignoring variable data: ->>> d=dict(ds.get_properties(o3_uid, byte_arrays=True)) ->>> del d['uid'], d['timestamp'] ->>> d -{dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/html', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest3', variant_level=1)} - - -Find entries using "known" metadata: ->>> sorted(ds.find({'mime_type': ['text/plain']}, ['title', 'activity', 'mime_type', 'tags'], byte_arrays=True)[0]) -[dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest1', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/plain', variant_level=1), dbus.String(u'tags'): dbus.ByteArray('bar baz', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 1 updated', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest1', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/plain', variant_level=1), dbus.String(u'tags'): dbus.ByteArray('foo', variant_level=1)}, signature=dbus.Signature('sv'))] ->>> sorted(ds.find({'mime_type': ['text/html']}, ['title', 'activity', 'mime_type', 'tags'], byte_arrays=True)[0]) -[dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/html', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest3', variant_level=1)}, signature=dbus.Signature('sv'))] ->>> sorted(ds.find({'uid': o3_uid}, ['title', 'activity', 'mime_type'], byte_arrays=True)[0]) -[dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/html', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest3', variant_level=1)}, signature=dbus.Signature('sv'))] ->>> sorted(ds.find({'timestamp': (9000, 11000)}, ['title', 'activity', 'mime_type'], byte_arrays=True)[0]) -[dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/html', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest3', variant_level=1)}, signature=dbus.Signature('sv'))] - -Find entries using "unknown" metadata (=> returns all entries): ->>> sorted(ds.find({'title': 'DS test object 2'}, ['title', 'activity', 'mime_type', 'tags'], byte_arrays=True)[0]) -[dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/html', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest3', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest1', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/plain', variant_level=1), dbus.String(u'tags'): dbus.ByteArray('bar baz', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 1 updated', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest1', variant_level=1), dbus.String(u'mime_type'): dbus.ByteArray('text/plain', variant_level=1), dbus.String(u'tags'): dbus.ByteArray('foo', variant_level=1)}, signature=dbus.Signature('sv'))] - -Specify a sort order: ->>> ds.find({'order_by': ['+title']}, ['title', 'activity'], byte_arrays=True)[0] -dbus.Array([dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest1', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest3', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 1 updated', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest1', variant_level=1)}, signature=dbus.Signature('sv'))], signature=dbus.Signature('a{sv}')) ->>> ds.find({'order_by': ['-title']}, ['title', 'activity'], byte_arrays=True)[0] -dbus.Array([dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 1 updated', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest1', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest1', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest3', variant_level=1)}, signature=dbus.Signature('sv'))], signature=dbus.Signature('a{sv}')) - -Delete an entry: ->>> ds.delete(o1_uid) ->>> sorted(ds.find({}, ['title', 'activity'], byte_arrays=True)[0]) -[dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest1', variant_level=1)}, signature=dbus.Signature('sv')), dbus.Dictionary({dbus.String(u'title'): dbus.ByteArray('DS test object 2', variant_level=1), dbus.String(u'activity'): dbus.ByteArray('org.silbe.DataStoreTest3', variant_level=1)}, signature=dbus.Signature('sv'))] - - -Create an entry with content: ->>> dog_content = 'The quick brown dog jumped over the lazy fox.' ->>> dog_props = {'title': 'dog/fox story', 'mime_type': 'text/plain'} ->>> dog_file = tempfile.NamedTemporaryFile() ->>> dog_file.write(dog_content) ->>> dog_file.flush() ->>> dog_uid = ds.create(dog_props, dog_file.name, False) - -Retrieve and verify the entry with content: ->>> dog_retrieved = ds.get_filename(dog_uid) ->>> assert(file(dog_retrieved).read() == dog_content) ->>> os.remove(dog_retrieved) - -Update the entry content: ->>> dog_content = 'The quick brown fox jumped over the lazy dog.' ->>> dog_file.seek(0) ->>> dog_file.write(dog_content) ->>> dog_file.flush() ->>> ds.update(dog_uid, dog_props, dog_file.name, False) - -Verify updated content: ->>> dog_retrieved = ds.get_filename(dog_uid) ->>> assert(file(dog_retrieved).read() == dog_content) ->>> os.remove(dog_retrieved) ->>> dog_file.close() diff --git a/tests/runalltests.py b/tests/runalltests.py index f539746..c39aa12 100755 --- a/tests/runalltests.py +++ b/tests/runalltests.py @@ -32,9 +32,10 @@ logging.basicConfig(level=logging.WARN, DOCTESTS = [ - 'basic_api.txt', + 'sugar_api_v2.txt', ] -DOCTEST_OPTIONS = doctest.ELLIPSIS | doctest.REPORT_ONLY_FIRST_FAILURE | REPORT_UDIFF +DOCTEST_OPTIONS = (doctest.ELLIPSIS | doctest.REPORT_ONLY_FIRST_FAILURE | + doctest.REPORT_UDIFF) DS_DBUS_SERVICE = 'org.silbe.GDataStore' DS_DBUS_INTERFACE = 'org.silbe.GDataStore' @@ -70,12 +71,12 @@ def setup(): environment['PATH'] = os.path.join(basedir, 'bin')+':'+os.environ['PATH'] servicedir = os.path.join(environment['HOME'], 'dbus-1', 'services') - servicepath = os.path.join(servicedir, 'org.silbe.GDataStore.service') + servicepath = os.path.join(servicedir, 'org.laptop.sugar.DataStore.service') os.makedirs(servicedir) servicefile = file(servicepath, 'w') servicefile.write(""" [D-BUS Service] - Name = org.silbe.GDataStore + Name = org.laptop.sugar.DataStore Exec = %s/bin/gdatastore-service """.replace(' ', '') % (basedir, )) servicefile.close() @@ -174,7 +175,7 @@ class TestSuiteWrapper(unittest.TestCase): def tearDown(self): self._kill_data_store() - self._clean_data_store() + #self._clean_data_store() def _kill_data_store(self): pgrep = subprocess.Popen(['pgrep', '-g', os.environ['DBUS_PID'], @@ -197,7 +198,7 @@ class TestSuiteWrapper(unittest.TestCase): self._loop.quit() def _clean_data_store(self): - shutil.rmtree(os.path.expanduser('~/.gdatastore')) + shutil.rmtree(os.path.expanduser('~/.gdatastore/git')) class TimedTestResult(unittest._TextTestResult): diff --git a/tests/sugar_api_v2.txt b/tests/sugar_api_v2.txt new file mode 100644 index 0000000..a2846eb --- /dev/null +++ b/tests/sugar_api_v2.txt @@ -0,0 +1,134 @@ +>>> import os +>>> import tempfile +>>> import time + +Define some helper functions +>>> def test_unique(items): +... return not [True for e in items if items.count(e) > 1] +>>> def to_native(value): +... if isinstance(value, list): +... return [to_native(e) for e in value] +... elif isinstance(value, dict): +... return dict([(to_native(k), to_native(v)) for k, v in value.items()]) +... elif isinstance(value, unicode): +... return unicode(value) +... elif isinstance(value, str): +... return str(value) +... return value + + +Connect to datastore using DBus and wait for it to get ready: +>>> import dbus +>>> DS_DBUS_SERVICE = 'org.laptop.sugar.DataStore' +>>> DS_DBUS_INTERFACE = 'org.laptop.sugar.DataStore' +>>> DS_DBUS_PATH = '/org/laptop/sugar/DataStore' +>>> bus = dbus.SessionBus() +>>> ds = dbus.Interface(bus.get_object(DS_DBUS_SERVICE, DS_DBUS_PATH), DS_DBUS_INTERFACE) + + +Make sure we're starting from an empty datastore: +>>> assert ds.find({}, [], byte_arrays=True) == ([], 0) + + +Create something to play with: +>>> o1_uid = ds.create({'title': 'DS test object 1', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest1'}, '', False) +>>> assert isinstance(o1_uid, basestring) +>>> o2_uid = ds.create({'title': 'DS test object 2', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest2'}, '', False) +>>> assert isinstance(o2_uid, basestring) +>>> o3_uid = ds.create({'title': 'DS test object 3', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest2'}, '', False) +>>> assert isinstance(o3_uid, basestring) +>>> assert test_unique([o1_uid, o2_uid, o3_uid]) + + +Check everything is there: +>>> assert sorted(to_native(ds.find({}, ['title', 'activity'], byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 1', u'activity': 'org.sugarlabs.DataStoreTest1'}, +... {u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest2'}, +... {u'title': 'DS test object 3', u'activity': 'org.sugarlabs.DataStoreTest2'}] +>>> ds.get_filename(o1_uid, byte_arrays=True) +dbus.String(u'') +>>> ds.get_filename(o2_uid, byte_arrays=True) +dbus.String(u'') +>>> ds.get_filename(o3_uid, byte_arrays=True) +dbus.String(u'') + + + +Test get_uniquevaluesfor(). +>>> sorted(ds.get_uniquevaluesfor('activity', {})) +[dbus.String(u'org.sugarlabs.DataStoreTest1'), dbus.String(u'org.sugarlabs.DataStoreTest2')] + + +Change some entries: +>>> ds.update(o1_uid, {'title': 'DS test object 1 updated', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest1', 'tags': 'foo'}, '', False) +>>> ds.update(o2_uid, {'title': 'DS test object 2', 'mime_type': 'text/plain', 'activity': 'org.sugarlabs.DataStoreTest1', 'tags': 'bar baz'}, '', False) +>>> ds.update(o3_uid, {'title': 'DS test object 2', 'mime_type': 'text/html', 'activity': 'org.sugarlabs.DataStoreTest3', 'timestamp': 10000}, '', False) +>>> assert sorted(to_native(ds.find({}, ['title', 'activity'], byte_arrays=True)[0])) == \ +... [{u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 1 updated'}, +... {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 2'}, +... {u'activity': 'org.sugarlabs.DataStoreTest3', u'title': 'DS test object 2'}] + +Retrieve metadata for a single entry, ignoring variable data: +>>> d=dict(ds.get_properties(o3_uid, byte_arrays=True)) +>>> del d['uid'], d['timestamp'], d['creation_time'] +>>> assert to_native(d) == {u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'} + + +Find entries using "standard" metadata: +>>> assert sorted(to_native(ds.find({'mime_type': ['text/plain']}, ['title', 'activity', 'mime_type', 'tags'], byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest1', u'mime_type': 'text/plain', u'tags': 'bar baz'}, +... {u'title': 'DS test object 1 updated', u'activity': 'org.sugarlabs.DataStoreTest1', u'mime_type': 'text/plain', u'tags': 'foo'}] +>>> assert sorted(to_native(ds.find({'mime_type': ['text/html']}, ['title', 'activity', 'mime_type', 'tags'], byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'}] +>>> assert sorted(to_native(ds.find({'uid': o3_uid}, ['title', 'activity', 'mime_type'], byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'}] +>>> assert sorted(to_native(ds.find({'timestamp': (9000, 11000)}, ['title', 'activity', 'mime_type'], byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'}] + +Find entries using "non-standard" metadata (only works with dict-based queries or prefixed Xapian query strings): +>>> assert sorted(to_native(ds.find({'title': 'DS test object 2'}, ['title', 'activity', 'mime_type', 'tags'], byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'mime_type': 'text/html', u'activity': 'org.sugarlabs.DataStoreTest3'}, +... {u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest1', u'mime_type': 'text/plain', u'tags': 'bar baz'}] + +You can specify a (primary) sort order. Please note that the secondary sort order is undefined / implementation-dependent. +>>> assert to_native(ds.find({'order_by': ['+title']}, ['title', 'activity'], byte_arrays=True)[0]) == \ +... [{u'activity': 'org.sugarlabs.DataStoreTest3', u'title': 'DS test object 2'}, +... {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 2'}, +... {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 1 updated'}] +>>> assert to_native(ds.find({'order_by': ['-title']}, ['title', 'activity'], byte_arrays=True)[0]) == \ +... [{u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 1 updated'}, +... {u'activity': 'org.sugarlabs.DataStoreTest1', u'title': 'DS test object 2'}, +... {u'activity': 'org.sugarlabs.DataStoreTest3', u'title': 'DS test object 2'}] + +Delete an entry: +>>> ds.delete(o1_uid) +>>> assert sorted(to_native(ds.find({}, ['title', 'activity'], byte_arrays=True)[0])) == \ +... [{u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest1'}, +... {u'title': 'DS test object 2', u'activity': 'org.sugarlabs.DataStoreTest3'}] + + +Create an entry with content: +>>> dog_content = 'The quick brown dog jumped over the lazy fox.' +>>> dog_props = {'title': 'dog/fox story', 'mime_type': 'text/plain'} +>>> dog_file = tempfile.NamedTemporaryFile() +>>> dog_file.write(dog_content) +>>> dog_file.flush() +>>> dog_uid = ds.create(dog_props, dog_file.name, False) + +Retrieve and verify the entry with content: +>>> dog_retrieved = ds.get_filename(dog_uid) +>>> assert(file(dog_retrieved).read() == dog_content) +>>> os.remove(dog_retrieved) + +Update the entry content: +>>> dog_content = 'The quick brown fox jumped over the lazy dog.' +>>> dog_file.seek(0) +>>> dog_file.write(dog_content) +>>> dog_file.flush() +>>> ds.update(dog_uid, dog_props, dog_file.name, False) + +Verify updated content: +>>> dog_retrieved = ds.get_filename(dog_uid) +>>> assert(file(dog_retrieved).read() == dog_content) +>>> os.remove(dog_retrieved) +>>> dog_file.close() diff --git a/tests/test_massops.py b/tests/test_massops.py index ec3bef6..b14d7ac 100644 --- a/tests/test_massops.py +++ b/tests/test_massops.py @@ -9,12 +9,17 @@ import time import unittest -DS_DBUS_SERVICE = 'org.silbe.GDataStore' -DS_DBUS_INTERFACE = 'org.silbe.GDataStore' -DS_DBUS_PATH = '/org/silbe/GDataStore' +DS_DBUS_SERVICE = 'org.laptop.sugar.DataStore' +DS_DBUS_INTERFACE = 'org.laptop.sugar.DataStore' +DS_DBUS_PATH = '/org/laptop/sugar/DataStore' NUM_RUNS = int(os.environ.get('MASSOPS_RUNS', '100')) IGNORE_PROPERTIES = [ + 'checksum', + 'creation_time', 'number', + 'timestamp', + 'uid', + 'version_id', ] @@ -29,17 +34,17 @@ class MassOpsTestCase(unittest.TestCase): """Large number of operations intended for measuring performance.""" def setUp(self): - # pylint: disable-msg=C0103 + # pylint: disable=C0103 self._bus = dbus.SessionBus() self._datastore = dbus.Interface(self._bus.get_object(DS_DBUS_SERVICE, - DS_DBUS_PATH), DS_DBUS_INTERFACE) + DS_DBUS_PATH), DS_DBUS_INTERFACE) _create_properties = { 'title': 'DS test object', 'mime_type': 'text/plain', - 'activity': 'org.silbe.GDataStoreTest1', + 'activity': 'org.sugarlabs.DataStoreTest1', } - _create_content = 'Foo bar\n'*1000 + _create_content = 'Foo bar\n' * 1000 def test_create(self): """Run create() lots of times to create new objects.""" @@ -57,17 +62,18 @@ class MassOpsTestCase(unittest.TestCase): def test_find_all(self): """Run find() to list all entries.""" entries, total_count = self._datastore.find({}, ['number'], - byte_arrays=True) + byte_arrays=True) self.assertEquals(total_count, NUM_RUNS) self.assertEquals(total_count, len(entries)) for position, entry in enumerate(entries): - self.assertEquals(int(entry['number']), NUM_RUNS-position-1) + self.assertEquals(int(entry['number']), NUM_RUNS - position - 1) @repeat def test_find_all_reverse_time(self): """Run find() to list all entries in reverse chronological order.""" - entries, total_count = self._datastore.find({'order_by': - ['-timestamp']}, ['number'], byte_arrays=True) + entries, total_count = \ + self._datastore.find({'order_by': ['-timestamp']}, ['number'], + byte_arrays=True) self.assertEquals(total_count, NUM_RUNS) self.assertEquals(total_count, len(entries)) for position, entry in enumerate(entries): @@ -76,16 +82,18 @@ class MassOpsTestCase(unittest.TestCase): @repeat def test_find_all_title(self): """Run find() to list all entries ordered by title.""" - entries, total_count = self._datastore.find({'order_by': - ['+title']}, ['tree_id'], byte_arrays=True) + entries, total_count = \ + self._datastore.find({'order_by': ['+title']}, ['tree_id'], + byte_arrays=True) self.assertEquals(total_count, NUM_RUNS) self.assertEquals(total_count, len(entries)) @repeat def test_find_all_reverse_title(self): """Run find() to list all entries ordered by title (reversed).""" - entries, total_count = self._datastore.find({'order_by': - ['-title']}, ['tree_id'], byte_arrays=True) + entries, total_count = \ + self._datastore.find({'order_by': ['-title']}, ['tree_id'], + byte_arrays=True) self.assertEquals(total_count, NUM_RUNS) self.assertEquals(total_count, len(entries)) @@ -94,21 +102,24 @@ class MassOpsTestCase(unittest.TestCase): """Run find() to list all entries in small chunks.""" chunk_size = 30 for chunk_start in range(0, NUM_RUNS, 30): - entries, total_count = self._datastore.find({ - 'offset': chunk_start, 'limit': chunk_size}, - ['number'], byte_arrays=True) + entries, total_count = \ + self._datastore.find({'offset': chunk_start, + 'limit': chunk_size}, ['number'], + byte_arrays=True) self.assertEquals(len(entries), - min(chunk_size, NUM_RUNS-chunk_start)) + min(chunk_size, NUM_RUNS - chunk_start)) self.assertEquals(total_count, NUM_RUNS) for position, entry in enumerate(entries): self.assertEquals(int(entry['number']), - NUM_RUNS-(chunk_start+position)-1) + NUM_RUNS - (chunk_start + position) - 1) def test_get_properties(self): """Run get_properties() on all entries and verify result.""" for entry in self._datastore.find({}, ['uid'], byte_arrays=True)[0]: - properties = self._datastore.get_properties(entry['uid'], - byte_arrays=True) + properties = \ + self._datastore.get_properties(entry['uid'], byte_arrays=True) + self.assertEquals(properties.pop('filesize'), + str(len(self._create_content))) self._filter_properties(properties) self.assertEquals(properties, self._create_properties) @@ -116,7 +127,7 @@ class MassOpsTestCase(unittest.TestCase): """Run get_filename() on all entries and verify content.""" for entry in self._datastore.find({}, ['uid'], byte_arrays=True)[0]: filename = self._datastore.get_filename(entry['uid'], - byte_arrays=True) + byte_arrays=True) try: self.assertEquals(file(filename).read(), self._create_content) finally: @@ -125,9 +136,9 @@ class MassOpsTestCase(unittest.TestCase): _update_properties = { 'title': 'DS test object (updated)', 'mime_type': 'text/plain', - 'activity': 'org.silbe.GDataStoreTest1', + 'activity': 'org.sugarlabs.DataStoreTest1', } - _update_content = 'Foo bar baz\n'*1000 + _update_content = 'Foo bar baz\n' * 1000 def test_update(self): """Update the content of all existing entries""" @@ -136,13 +147,17 @@ class MassOpsTestCase(unittest.TestCase): content_file.flush() for entry in self._datastore.find({}, ['uid'], byte_arrays=True)[0]: self._datastore.update(entry['uid'], self._update_properties, - content_file.name, False) + content_file.name, False) def test_update_verify(self): - """Verify test_update() has changed content and metadata of all entries.""" + """Verify test_update() has changed content and metadata of all + entries. + """ for entry in self._datastore.find({}, [], byte_arrays=True)[0]: filename = self._datastore.get_filename(entry['uid'], - byte_arrays=True) + byte_arrays=True) + self.assertEquals(entry.pop('filesize'), + str(len(self._update_content))) self._filter_properties(entry) try: self.assertEquals(entry, self._update_properties) -- cgit v0.9.1