Merge commit 'refs/top-bases/t/rainbow-0.8' into t/rainbow-0.8

9 files changed, 132 insertions, 29 deletions

diff --git a/AUTHORS b/AUTHORS
index a2c2720..815ace9 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -1,7 +1,7 @@
 Contributors
-============
-
+~~~~~~~~~~~~
 Aleksey Lim <alsroot@member.fsf.org>
+Andrés Ambrois <andresambrois@gmail.com>
 Benjamin Saller <bcsaller@objectrealms.net>
 Bernie Innocenti <bernie@codewiz.org>
 Bert Freudenberg <bert@freudenbergs.de>
@@ -13,12 +13,12 @@ Simon Schampijer <simon@schampijer.de>
 Tomeu Vizoso <tomeu@sugarlabs.org>
 Wade Brainerd <wadetb@gmail.com>
 
-Past maintainers
-================
 
+Past maintainers
+~~~~~~~~~~~~~~~~
 Tomeu Vizoso <tomeu@sugarlabs.org>
 
-Current maintainers
-===================
 
+Current maintainers
+~~~~~~~~~~~~~~~~~~~
 Aleksey Lim <alsroot@member.fsf.org>
diff --git a/NEWS b/NEWS
index 0a765be..f5a9eb6 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,8 @@
+v0.89.3 2010-08-24
+* New metadata fields, creation_time and filesize to support sorting
+  in the Journal #1915
+* Create target directory before importing previews (carrott) #2149
+
 v0.89.2 2010-08-04
 * sl#2132: reduce _FLUSH_TIMEOUT to 5 seconds
 * Set index_updated flag on ds shutting down #2095
diff --git a/README b/README
index b29cebb..b6b4ed4 100644
--- a/README
+++ b/README
@@ -1,13 +1,11 @@
 About
-=====
-
+~~~~~
 Provides activities with a way to store data and metadata and the journal
 with means for querying, including full text search.
 
 
 Resources
-=========
-
+~~~~~~~~~
 Code Repository
     http://git.sugarlabs.org/projects/sugar-datastore/
 
@@ -19,8 +17,7 @@ Home Page
 
 
 Storage format history
-======================
-
+~~~~~~~~~~~~~~~~~~~~~~
 0   0.82.x
     Initial format
 
@@ -38,4 +35,9 @@ Storage format history
     migration to version 2 (SL#1787)
 
 5   not-mainstream
-    test versioning support (version bump for SL#1787)
+    pre v6 testing
+
+6   0.90
+    new metadata fields:
+    - creation_time, time of ds entry creation in seconds since the epoch
+    - filesize, size of ds entry data file in bytes
diff --git a/autogen.sh b/autogen.sh
index 1cd5db4..b02ffac 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -2,4 +2,4 @@
 export ACLOCAL="aclocal -I m4"
 
 autoreconf -i
-./configure "$@"
+./configure --enable-maintainer-mode "$@"
diff --git a/configure.ac b/configure.ac
index 6ccea2e..5eb3d9a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,4 +1,4 @@
-AC_INIT([sugar-datastore],[0.89.2],[],[sugar-datastore])
+AC_INIT([sugar-datastore],[0.89.3],[],[sugar-datastore])
 
 AC_PREREQ([2.59])
 
diff --git a/src/carquinyol/datastore.py b/src/carquinyol/datastore.py
index 82a6207..86d5b02 100644
--- a/src/carquinyol/datastore.py
+++ b/src/carquinyol/datastore.py
@@ -56,7 +56,7 @@ class DataStore(dbus.service.Object):
                                         allow_replacement=False)
         dbus.service.Object.__init__(self, bus_name, DS_OBJECT_PATH)
 
-        migrated = self._migrate()
+        migrated, initiated = self._open_layout()
 
         self._metadata_store = MetadataStore()
         self._file_store = FileStore()
@@ -75,26 +75,36 @@ class DataStore(dbus.service.Object):
             self._rebuild_index()
             return
 
-        if not self._index_store.index_updated:
+        if initiated:
+            logging.debug('Initiate datastore')
+            self._index_store.flush()
+        elif not self._index_store.index_updated:
             logging.debug('Index is not up-to-date, will update')
             self._update_index()
 
-    def _migrate(self):
-        """Check version of data store on disk and migrate if necessary.
+    def _open_layout(self):
+        """Open layout manager, check version of data store on disk and
+        migrate if necessary.
 
-        Returns True if migration was done and an index rebuild is required,
-        False otherwise.
+        Returns a pair of booleans. For the first, True if migration was done
+        and an index rebuild is required. For the second, True if datastore was
+        just initiated.
         """
         layout_manager = layoutmanager.get_instance()
+
+        if layout_manager.is_empty():
+            layout_manager.set_version(layoutmanager.CURRENT_LAYOUT_VERSION)
+            return False, True
+
         old_version = layout_manager.get_version()
         if old_version == layoutmanager.CURRENT_LAYOUT_VERSION:
-            return False
+            return False, False
 
         if old_version == 0:
             migration.migrate_from_0()
 
         layout_manager.set_version(layoutmanager.CURRENT_LAYOUT_VERSION)
-        return True
+        return True, False
 
     def _rebuild_index(self):
         """Remove and recreate index."""
@@ -121,7 +131,26 @@ class DataStore(dbus.service.Object):
 
             if not self._index_store.contains(uid):
                 try:
+                    update_metadata = False
                     props = self._metadata_store.retrieve(uid)
+                    if 'filesize' not in props:
+                        path = self._file_store.get_file_path(uid)
+                        if os.path.exists(path):
+                            props['filesize'] = os.stat(path).st_size
+                            update_metadata = True
+                    if 'creation_time' not in props:
+                        if 'ctime' in props:
+                            try:
+                                props['creation_time'] = time.mktime(
+                                        time.strptime(props['ctime'],
+                                            migration.DATE_FORMAT))
+                            except (TypeError, ValueError):
+                                pass
+                        if 'creation_time' not in props:
+                            props['creation_time'] = props['timestamp']
+                        update_metadata = True
+                    if update_metadata:
+                        self._metadata_store.store(uid, props)
                     self._index_store.store(uid, props)
                 except Exception:
                     logging.exception('Error processing %r', uid)
@@ -159,6 +188,23 @@ class DataStore(dbus.service.Object):
         if not props.get('timestamp', ''):
             props['timestamp'] = int(time.time())
 
+        # FIXME: Support for the deprecated ctime property. Remove in 0.92.
+        if 'ctime' in props:
+            try:
+                props['creation_time'] = time.mktime(time.strptime(
+                    migration.DATE_FORMAT, props['ctime']))
+            except (TypeError, ValueError):
+                pass
+
+        if 'creation_time' not in props:
+            props['creation_time'] = props['timestamp']
+
+        if os.path.exists(file_path):
+            stat = os.stat(file_path)
+            props['filesize'] = stat.st_size
+        else:
+            props['filesize'] = 0
+
         self._metadata_store.store(uid, props)
         self._index_store.store(uid, props)
         self._file_store.store(uid, file_path, transfer_ownership,
@@ -195,6 +241,26 @@ class DataStore(dbus.service.Object):
         if not props.get('timestamp', ''):
             props['timestamp'] = int(time.time())
 
+        # FIXME: Support for the deprecated ctime property. Remove in 0.92.
+        if 'ctime' in props:
+            try:
+                props['creation_time'] = time.mktime(time.strptime(
+                    migration.DATE_FORMAT, props['ctime']))
+            except (TypeError, ValueError):
+                pass
+
+        if 'creation_time' not in props:
+            props['creation_time'] = props['timestamp']
+
+        if file_path:
+            # Empty file_path means skipping storage stage, see filestore.py
+            # TODO would be more useful to update filesize after real file save
+            if os.path.exists(file_path):
+                stat = os.stat(file_path)
+                props['filesize'] = stat.st_size
+            else:
+                props['filesize'] = 0
+
         self._metadata_store.store(uid, props)
         self._index_store.store(uid, props)
 
diff --git a/src/carquinyol/indexstore.py b/src/carquinyol/indexstore.py
index 62b843b..8a3de30 100644
--- a/src/carquinyol/indexstore.py
+++ b/src/carquinyol/indexstore.py
@@ -28,6 +28,9 @@ from carquinyol.layoutmanager import MAX_QUERY_LIMIT
 _VALUE_UID = 0
 _VALUE_TIMESTAMP = 1
 _VALUE_TITLE = 2
+# 3 reserved for version support
+_VALUE_FILESIZE = 4
+_VALUE_CREATION_TIME = 5
 
 _PREFIX_NONE = 'N'
 _PREFIX_FULL_VALUE = 'F'
@@ -57,6 +60,8 @@ _QUERY_TERM_MAP = {
 
 _QUERY_VALUE_MAP = {
     'timestamp': {'number': _VALUE_TIMESTAMP, 'type': float},
+    'filesize': {'number': _VALUE_FILESIZE, 'type': int},
+    'creation_time': {'number': _VALUE_CREATION_TIME, 'type': float},
 }
 
 
@@ -66,6 +71,22 @@ class TermGenerator (xapian.TermGenerator):
         document.add_value(_VALUE_TIMESTAMP,
             xapian.sortable_serialise(float(properties['timestamp'])))
         document.add_value(_VALUE_TITLE, properties.get('title', '').strip())
+        if 'filesize' in properties:
+            try:
+                document.add_value(_VALUE_FILESIZE,
+                    xapian.sortable_serialise(int(properties['filesize'])))
+            except (ValueError, TypeError):
+                logging.debug('Invalid value for filesize property: %s',
+                              properties['filesize'])
+        if 'creation_time' in properties:
+            try:
+                document.add_value(
+                    _VALUE_CREATION_TIME, xapian.sortable_serialise(
+                        float(properties['creation_time'])))
+            except (ValueError, TypeError):
+                logging.debug('Invalid value for creation_time property: %s',
+                              properties['creation_time'])
+
 
         self.set_document(document)
 
@@ -286,6 +307,14 @@ class IndexStore(object):
             enquire.set_sort_by_value(_VALUE_TITLE, True)
         elif order_by == '-title':
             enquire.set_sort_by_value(_VALUE_TITLE, False)
+        elif order_by == '+filesize':
+            enquire.set_sort_by_value(_VALUE_FILESIZE, True)
+        elif order_by == '-filesize':
+            enquire.set_sort_by_value(_VALUE_FILESIZE, False)
+        elif order_by == '+creation_time':
+            enquire.set_sort_by_value(_VALUE_CREATION_TIME, True)
+        elif order_by == '-creation_time':
+            enquire.set_sort_by_value(_VALUE_CREATION_TIME, False)
         else:
             logging.warning('Unsupported property for sorting: %s', order_by)
 
diff --git a/src/carquinyol/layoutmanager.py b/src/carquinyol/layoutmanager.py
index 5c67203..335614f 100644
--- a/src/carquinyol/layoutmanager.py
+++ b/src/carquinyol/layoutmanager.py
@@ -18,7 +18,7 @@ import os
 import logging
 
 MAX_QUERY_LIMIT = 40960
-CURRENT_LAYOUT_VERSION = 4
+CURRENT_LAYOUT_VERSION = 6
 
 class LayoutManager(object):
     """Provide the logic about how entries are stored inside the datastore
@@ -88,7 +88,7 @@ class LayoutManager(object):
                         uids.append(g)
         return uids
 
-    def _is_empty(self):
+    def is_empty(self):
         """Check if there is any existing entry.
 
         All data store layout versions are handled. Will err on the safe
diff --git a/src/carquinyol/migration.py b/src/carquinyol/migration.py
index 95ee391..686902f 100644
--- a/src/carquinyol/migration.py
+++ b/src/carquinyol/migration.py
@@ -44,6 +44,11 @@ def migrate_from_0():
             continue
 
         logging.debug('Migrating entry %r', uid)
+
+        new_entry_dir = layoutmanager.get_instance().get_metadata_path(uid)
+        if not os.path.exists(new_entry_dir):
+            os.makedirs(new_entry_dir)
+
         try:
             _migrate_metadata(root_path, old_root_path, uid)
             _migrate_file(root_path, old_root_path, uid)
@@ -59,10 +64,7 @@ def migrate_from_0():
 
 
 def _migrate_metadata(root_path, old_root_path, uid):
-    dir_path = layoutmanager.get_instance().get_entry_path(uid)
     metadata_path = layoutmanager.get_instance().get_metadata_path(uid)
-    os.makedirs(metadata_path)
-
     old_metadata_path = os.path.join(old_root_path, uid + '.metadata')
     metadata = cjson.decode(open(old_metadata_path, 'r').read())
 
@@ -97,7 +99,6 @@ def _migrate_file(root_path, old_root_path, uid):
 
 
 def _migrate_preview(root_path, old_root_path, uid):
-    dir_path = layoutmanager.get_instance().get_entry_path(uid)
     metadata_path = layoutmanager.get_instance().get_metadata_path(uid)
     os.rename(os.path.join(old_root_path, 'preview', uid),
               os.path.join(metadata_path, 'preview'))