From 6dc23a1e9bad86ca0a83b94e8e71666ee21ccdae Mon Sep 17 00:00:00 2001 From: Sascha Silbe Date: Wed, 08 Sep 2010 17:25:29 +0000 Subject: Merge commit 'refs/top-bases/t/rainbow-0.8' into t/rainbow-0.8 --- diff --git a/AUTHORS b/AUTHORS index a2c2720..815ace9 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,7 +1,7 @@ Contributors -============ - +~~~~~~~~~~~~ Aleksey Lim +Andrés Ambrois Benjamin Saller Bernie Innocenti Bert Freudenberg @@ -13,12 +13,12 @@ Simon Schampijer Tomeu Vizoso Wade Brainerd -Past maintainers -================ +Past maintainers +~~~~~~~~~~~~~~~~ Tomeu Vizoso -Current maintainers -=================== +Current maintainers +~~~~~~~~~~~~~~~~~~~ Aleksey Lim diff --git a/NEWS b/NEWS index 0a765be..f5a9eb6 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,8 @@ +v0.89.3 2010-08-24 +* New metadata fields, creation_time and filesize to support sorting + in the Journal #1915 +* Create target directory before importing previews (carrott) #2149 + v0.89.2 2010-08-04 * sl#2132: reduce _FLUSH_TIMEOUT to 5 seconds * Set index_updated flag on ds shutting down #2095 diff --git a/README b/README index b29cebb..b6b4ed4 100644 --- a/README +++ b/README @@ -1,13 +1,11 @@ About -===== - +~~~~~ Provides activities with a way to store data and metadata and the journal with means for querying, including full text search. Resources -========= - +~~~~~~~~~ Code Repository http://git.sugarlabs.org/projects/sugar-datastore/ @@ -19,8 +17,7 @@ Home Page Storage format history -====================== - +~~~~~~~~~~~~~~~~~~~~~~ 0 0.82.x Initial format @@ -38,4 +35,9 @@ Storage format history migration to version 2 (SL#1787) 5 not-mainstream - test versioning support (version bump for SL#1787) + pre v6 testing + +6 0.90 + new metadata fields: + - creation_time, time of ds entry creation in seconds since the epoch + - filesize, size of ds entry data file in bytes diff --git a/autogen.sh b/autogen.sh index 1cd5db4..b02ffac 100755 --- a/autogen.sh +++ b/autogen.sh @@ -2,4 +2,4 @@ export ACLOCAL="aclocal -I m4" autoreconf -i -./configure "$@" +./configure --enable-maintainer-mode "$@" diff --git a/configure.ac b/configure.ac index 6ccea2e..5eb3d9a 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT([sugar-datastore],[0.89.2],[],[sugar-datastore]) +AC_INIT([sugar-datastore],[0.89.3],[],[sugar-datastore]) AC_PREREQ([2.59]) diff --git a/src/carquinyol/datastore.py b/src/carquinyol/datastore.py index 82a6207..86d5b02 100644 --- a/src/carquinyol/datastore.py +++ b/src/carquinyol/datastore.py @@ -56,7 +56,7 @@ class DataStore(dbus.service.Object): allow_replacement=False) dbus.service.Object.__init__(self, bus_name, DS_OBJECT_PATH) - migrated = self._migrate() + migrated, initiated = self._open_layout() self._metadata_store = MetadataStore() self._file_store = FileStore() @@ -75,26 +75,36 @@ class DataStore(dbus.service.Object): self._rebuild_index() return - if not self._index_store.index_updated: + if initiated: + logging.debug('Initiate datastore') + self._index_store.flush() + elif not self._index_store.index_updated: logging.debug('Index is not up-to-date, will update') self._update_index() - def _migrate(self): - """Check version of data store on disk and migrate if necessary. + def _open_layout(self): + """Open layout manager, check version of data store on disk and + migrate if necessary. - Returns True if migration was done and an index rebuild is required, - False otherwise. + Returns a pair of booleans. For the first, True if migration was done + and an index rebuild is required. For the second, True if datastore was + just initiated. """ layout_manager = layoutmanager.get_instance() + + if layout_manager.is_empty(): + layout_manager.set_version(layoutmanager.CURRENT_LAYOUT_VERSION) + return False, True + old_version = layout_manager.get_version() if old_version == layoutmanager.CURRENT_LAYOUT_VERSION: - return False + return False, False if old_version == 0: migration.migrate_from_0() layout_manager.set_version(layoutmanager.CURRENT_LAYOUT_VERSION) - return True + return True, False def _rebuild_index(self): """Remove and recreate index.""" @@ -121,7 +131,26 @@ class DataStore(dbus.service.Object): if not self._index_store.contains(uid): try: + update_metadata = False props = self._metadata_store.retrieve(uid) + if 'filesize' not in props: + path = self._file_store.get_file_path(uid) + if os.path.exists(path): + props['filesize'] = os.stat(path).st_size + update_metadata = True + if 'creation_time' not in props: + if 'ctime' in props: + try: + props['creation_time'] = time.mktime( + time.strptime(props['ctime'], + migration.DATE_FORMAT)) + except (TypeError, ValueError): + pass + if 'creation_time' not in props: + props['creation_time'] = props['timestamp'] + update_metadata = True + if update_metadata: + self._metadata_store.store(uid, props) self._index_store.store(uid, props) except Exception: logging.exception('Error processing %r', uid) @@ -159,6 +188,23 @@ class DataStore(dbus.service.Object): if not props.get('timestamp', ''): props['timestamp'] = int(time.time()) + # FIXME: Support for the deprecated ctime property. Remove in 0.92. + if 'ctime' in props: + try: + props['creation_time'] = time.mktime(time.strptime( + migration.DATE_FORMAT, props['ctime'])) + except (TypeError, ValueError): + pass + + if 'creation_time' not in props: + props['creation_time'] = props['timestamp'] + + if os.path.exists(file_path): + stat = os.stat(file_path) + props['filesize'] = stat.st_size + else: + props['filesize'] = 0 + self._metadata_store.store(uid, props) self._index_store.store(uid, props) self._file_store.store(uid, file_path, transfer_ownership, @@ -195,6 +241,26 @@ class DataStore(dbus.service.Object): if not props.get('timestamp', ''): props['timestamp'] = int(time.time()) + # FIXME: Support for the deprecated ctime property. Remove in 0.92. + if 'ctime' in props: + try: + props['creation_time'] = time.mktime(time.strptime( + migration.DATE_FORMAT, props['ctime'])) + except (TypeError, ValueError): + pass + + if 'creation_time' not in props: + props['creation_time'] = props['timestamp'] + + if file_path: + # Empty file_path means skipping storage stage, see filestore.py + # TODO would be more useful to update filesize after real file save + if os.path.exists(file_path): + stat = os.stat(file_path) + props['filesize'] = stat.st_size + else: + props['filesize'] = 0 + self._metadata_store.store(uid, props) self._index_store.store(uid, props) diff --git a/src/carquinyol/indexstore.py b/src/carquinyol/indexstore.py index 62b843b..8a3de30 100644 --- a/src/carquinyol/indexstore.py +++ b/src/carquinyol/indexstore.py @@ -28,6 +28,9 @@ from carquinyol.layoutmanager import MAX_QUERY_LIMIT _VALUE_UID = 0 _VALUE_TIMESTAMP = 1 _VALUE_TITLE = 2 +# 3 reserved for version support +_VALUE_FILESIZE = 4 +_VALUE_CREATION_TIME = 5 _PREFIX_NONE = 'N' _PREFIX_FULL_VALUE = 'F' @@ -57,6 +60,8 @@ _QUERY_TERM_MAP = { _QUERY_VALUE_MAP = { 'timestamp': {'number': _VALUE_TIMESTAMP, 'type': float}, + 'filesize': {'number': _VALUE_FILESIZE, 'type': int}, + 'creation_time': {'number': _VALUE_CREATION_TIME, 'type': float}, } @@ -66,6 +71,22 @@ class TermGenerator (xapian.TermGenerator): document.add_value(_VALUE_TIMESTAMP, xapian.sortable_serialise(float(properties['timestamp']))) document.add_value(_VALUE_TITLE, properties.get('title', '').strip()) + if 'filesize' in properties: + try: + document.add_value(_VALUE_FILESIZE, + xapian.sortable_serialise(int(properties['filesize']))) + except (ValueError, TypeError): + logging.debug('Invalid value for filesize property: %s', + properties['filesize']) + if 'creation_time' in properties: + try: + document.add_value( + _VALUE_CREATION_TIME, xapian.sortable_serialise( + float(properties['creation_time']))) + except (ValueError, TypeError): + logging.debug('Invalid value for creation_time property: %s', + properties['creation_time']) + self.set_document(document) @@ -286,6 +307,14 @@ class IndexStore(object): enquire.set_sort_by_value(_VALUE_TITLE, True) elif order_by == '-title': enquire.set_sort_by_value(_VALUE_TITLE, False) + elif order_by == '+filesize': + enquire.set_sort_by_value(_VALUE_FILESIZE, True) + elif order_by == '-filesize': + enquire.set_sort_by_value(_VALUE_FILESIZE, False) + elif order_by == '+creation_time': + enquire.set_sort_by_value(_VALUE_CREATION_TIME, True) + elif order_by == '-creation_time': + enquire.set_sort_by_value(_VALUE_CREATION_TIME, False) else: logging.warning('Unsupported property for sorting: %s', order_by) diff --git a/src/carquinyol/layoutmanager.py b/src/carquinyol/layoutmanager.py index 5c67203..335614f 100644 --- a/src/carquinyol/layoutmanager.py +++ b/src/carquinyol/layoutmanager.py @@ -18,7 +18,7 @@ import os import logging MAX_QUERY_LIMIT = 40960 -CURRENT_LAYOUT_VERSION = 4 +CURRENT_LAYOUT_VERSION = 6 class LayoutManager(object): """Provide the logic about how entries are stored inside the datastore @@ -88,7 +88,7 @@ class LayoutManager(object): uids.append(g) return uids - def _is_empty(self): + def is_empty(self): """Check if there is any existing entry. All data store layout versions are handled. Will err on the safe diff --git a/src/carquinyol/migration.py b/src/carquinyol/migration.py index 95ee391..686902f 100644 --- a/src/carquinyol/migration.py +++ b/src/carquinyol/migration.py @@ -44,6 +44,11 @@ def migrate_from_0(): continue logging.debug('Migrating entry %r', uid) + + new_entry_dir = layoutmanager.get_instance().get_metadata_path(uid) + if not os.path.exists(new_entry_dir): + os.makedirs(new_entry_dir) + try: _migrate_metadata(root_path, old_root_path, uid) _migrate_file(root_path, old_root_path, uid) @@ -59,10 +64,7 @@ def migrate_from_0(): def _migrate_metadata(root_path, old_root_path, uid): - dir_path = layoutmanager.get_instance().get_entry_path(uid) metadata_path = layoutmanager.get_instance().get_metadata_path(uid) - os.makedirs(metadata_path) - old_metadata_path = os.path.join(old_root_path, uid + '.metadata') metadata = cjson.decode(open(old_metadata_path, 'r').read()) @@ -97,7 +99,6 @@ def _migrate_file(root_path, old_root_path, uid): def _migrate_preview(root_path, old_root_path, uid): - dir_path = layoutmanager.get_instance().get_entry_path(uid) metadata_path = layoutmanager.get_instance().get_metadata_path(uid) os.rename(os.path.join(old_root_path, 'preview', uid), os.path.join(metadata_path, 'preview')) -- cgit v0.9.1