rebuild index on migration (#1787)

Currently we force an index rescan on migration, but not a rebuild.
Any entry that's already in the index will _not_ be updated.
This patch fixes the migration logic to do a full index rebuild,
thus actually migrating the index.

Signed-off-by: Sascha Silbe <sascha-pgp@silbe.org>

4 files changed, 53 insertions, 30 deletions

diff --git a/README b/README
index ae5ec36..84da051 100644
--- a/README
+++ b/README
@@ -32,3 +32,11 @@ Storage format history
 
 3   not-mainstream
     test versioning support
+
+4   0.88, 0.86.2
+    version bump to force index rebuild that may have been missed during the
+    migration to version 2 (SL#1787)
+
+5   not-mainstream
+    test versioning support (version bump for SL#1787)
+
diff --git a/src/carquinyol/datastore.py b/src/carquinyol/datastore.py
index dc2e167..6824d90 100644
--- a/src/carquinyol/datastore.py
+++ b/src/carquinyol/datastore.py
@@ -56,41 +56,62 @@ class DataStore(dbus.service.Object):
                                         allow_replacement=False)
         dbus.service.Object.__init__(self, bus_name, DS_OBJECT_PATH)
 
-        layout_manager = layoutmanager.get_instance()
-        if layout_manager.get_version() == 0:
-            migration.migrate_from_0()
-            layout_manager.set_version(layoutmanager.CURRENT_LAYOUT_VERSION)
-            layout_manager.index_updated = False
-        elif layout_manager.get_version() == 1:
-            layout_manager.set_version(layoutmanager.CURRENT_LAYOUT_VERSION)
-            layout_manager.index_updated = False
+        migrated = self._migrate()
 
         self._metadata_store = MetadataStore()
-
+        self._file_store = FileStore()
+        self._optimizer = Optimizer(self._file_store, self._metadata_store)
         self._index_store = IndexStore()
+
+        if migrated:
+            self._rebuild_index()
+            return
+
         try:
             self._index_store.open_index()
         except Exception:
             logging.exception('Failed to open index, will rebuild')
-            layout_manager.index_updated = False
-            self._index_store.remove_index()
-            self._index_store.open_index()
-
-        self._file_store = FileStore()
+            self._rebuild_index()
+            return
 
         if not layout_manager.index_updated:
             logging.debug('Index is not up-to-date, will update')
-            self._rebuild_index()
+            self._update_index()
 
-        self._optimizer = Optimizer(self._file_store, self._metadata_store)
+    def _migrate(self):
+        """Check version of data store on disk and migrate if necessary.
+
+        Returns True if migration was done and an index rebuild is required,
+        False otherwise.
+        """
+        layout_manager = layoutmanager.get_instance()
+        old_version = layout_manager.get_version()
+        if old_version == layoutmanager.CURRENT_LAYOUT_VERSION:
+            return False
+
+        if old_version == 0:
+            migration.migrate_from_0()
+
+        layout_manager.set_version(layoutmanager.CURRENT_LAYOUT_VERSION)
+        return True
 
     def _rebuild_index(self):
+        """Remove and recreate index."""
+        layoutmanager.get_instance().index_updated = False
+        self._index_store.close_index()
+        self._index_store.remove_index()
+        self._index_store.open_index()
+        self._update_index()
+
+    def _update_index(self):
+        """Find entries that are not yet in the index and add them."""
         uids = layoutmanager.get_instance().find_all()
-        logging.debug('Going to update the index with uids %r', uids)
-        gobject.idle_add(lambda: self.__rebuild_index_cb(uids),
+        logging.debug('Going to update the index with object_ids %r',
+            uids)
+        gobject.idle_add(lambda: self.__update_index_cb(uids),
                             priority=gobject.PRIORITY_LOW)
 
-    def __rebuild_index_cb(self, uids):
+    def __update_index_cb(self, uids):
         if uids:
             uid = uids.pop()
 
@@ -200,10 +221,6 @@ class DataStore(dbus.service.Object):
                 uids, count = self._index_store.find(query)
             except Exception:
                 logging.exception('Failed to query index, will rebuild')
-                layoutmanager.get_instance().index_updated = False
-                self._index_store.close_index()
-                self._index_store.remove_index()
-                self._index_store.open_index()
                 self._rebuild_index()
 
         if not layoutmanager.get_instance().index_updated:
@@ -216,13 +233,7 @@ class DataStore(dbus.service.Object):
             if not os.path.exists(entry_path):
                 logging.warning(
                     'Inconsistency detected, returning all entries')
-
-                layoutmanager.get_instance().index_updated = False
-                self._index_store.close_index()
-                self._index_store.remove_index()
-                self._index_store.open_index()
                 self._rebuild_index()
-
                 return self._find_all(query, properties)
 
             metadata = self._metadata_store.retrieve(uid, properties)
diff --git a/src/carquinyol/indexstore.py b/src/carquinyol/indexstore.py
index 4dfd620..8a69334 100644
--- a/src/carquinyol/indexstore.py
+++ b/src/carquinyol/indexstore.py
@@ -220,6 +220,10 @@ class IndexStore(object):
         self._database = WritableDatabase(index_path, xapian.DB_CREATE_OR_OPEN)
 
     def close_index(self):
+        """Close index database if it is open."""
+        if not self._database:
+            return
+
         self._database.flush()
         self._database = None
 
diff --git a/src/carquinyol/layoutmanager.py b/src/carquinyol/layoutmanager.py
index 1f2dd4c..0b0b91a 100644
--- a/src/carquinyol/layoutmanager.py
+++ b/src/carquinyol/layoutmanager.py
@@ -18,7 +18,7 @@ import os
 import logging
 
 MAX_QUERY_LIMIT = 40960
-CURRENT_LAYOUT_VERSION = 2
+CURRENT_LAYOUT_VERSION = 4
 
 class LayoutManager(object):
     """Provide the logic about how entries are stored inside the datastore