Implement data migration from the older datastore

5 files changed, 90 insertions, 9 deletions

diff --git a/src/olpc/datastore/Makefile.am b/src/olpc/datastore/Makefile.am
index 43b6d14..6d6c2fe 100644
--- a/src/olpc/datastore/Makefile.am
+++ b/src/olpc/datastore/Makefile.am
@@ -5,7 +5,8 @@ datastore_PYTHON = 		\
 	filestore.py		\
 	indexstore.py		\
 	layoutmanager.py	\
-	metadatastore.py
+	metadatastore.py	\
+	migration.py
 
 AM_CPPFLAGS = 			\
 	$(WARN_CFLAGS)		\
diff --git a/src/olpc/datastore/datastore.py b/src/olpc/datastore/datastore.py
index 1b8af7a..60b29f5 100644
--- a/src/olpc/datastore/datastore.py
+++ b/src/olpc/datastore/datastore.py
@@ -19,6 +19,7 @@ import dbus
 import gobject
 
 from olpc.datastore import layoutmanager
+from olpc.datastore import migration
 from olpc.datastore.layoutmanager import MAX_QUERY_LIMIT
 from olpc.datastore.metadatastore import MetadataStore
 from olpc.datastore.indexstore import IndexStore
@@ -42,6 +43,12 @@ class DataStore(dbus.service.Object):
                                         allow_replacement=False)
         dbus.service.Object.__init__(self, bus_name, DS_OBJECT_PATH)
 
+        layout_manager = layoutmanager.get_instance()
+        if layout_manager.get_version() == 0:
+            migration.migrate_from_0()
+            layout_manager.set_version(1)
+            layout_manager.index_updated = False
+
         self._metadata_store = MetadataStore()
 
         self._index_store = IndexStore()
@@ -49,18 +56,19 @@ class DataStore(dbus.service.Object):
             self._index_store.open_index()
         except Exception, e:
             logging.error('Failed to open index, will rebuild: %r', e)
-            layoutmanager.get_instance().index_updated = False
+            layout_manager.index_updated = False
             self._index_store.remove_index()
             self._index_store.open_index()
 
         self._file_store = FileStore()
 
-        if not layoutmanager.get_instance().index_updated:
+        if not layout_manager.index_updated:
             logging.debug('Index is not up-to-date, will update')
             self._rebuild_index()
 
     def _rebuild_index(self):
         uids = layoutmanager.get_instance().find_all()
+        logging.debug('Going to update the index with uids %r' % uids)
         gobject.idle_add(lambda: self.__rebuild_index_cb(uids),
                             priority=gobject.PRIORITY_LOW)
 
diff --git a/src/olpc/datastore/layoutmanager.py b/src/olpc/datastore/layoutmanager.py
index a01139e..36c5811 100644
--- a/src/olpc/datastore/layoutmanager.py
+++ b/src/olpc/datastore/layoutmanager.py
@@ -7,9 +7,12 @@ class LayoutManager(object):
         profile = os.environ.get('SUGAR_PROFILE', 'default')
         base_dir = os.path.join(os.path.expanduser('~'), '.sugar', profile)
 
-        self._root_path = os.path.join(base_dir, 'datastore2')
+        self._root_path = os.path.join(base_dir, 'datastore')
+
+        if not os.path.exists(self._root_path):
+            os.makedirs(self._root_path)
+            self.set_version(1)
 
-        self._create_if_needed(self._root_path)
         self._create_if_needed(self.get_checksums_dir())
         self._create_if_needed(self.get_queue_path())
 
@@ -20,9 +23,24 @@ class LayoutManager(object):
         if not os.path.exists(path):
             os.makedirs(path)
 
+    def get_version(self):
+        version_path = os.path.join(self._root_path, 'version')
+        version = 0
+        if os.path.exists(version_path):
+            version = int(open(version_path, 'r').read())
+        return version
+
+    def set_version(self, version):
+        version_path = os.path.join(self._root_path, 'version')
+        open(version_path, 'w').write(str(version))
+
     def get_entry_path(self, uid):
+        # os.path.join() is just too slow
         return '%s/%s/%s' % (self._root_path, uid[:2], uid)
 
+    def get_root_path(self):
+        return self._root_path
+
     def get_index_path(self):
         return os.path.join(self._root_path, 'index')
 
@@ -50,8 +68,10 @@ class LayoutManager(object):
     def find_all(self):
         uids = []
         for f in os.listdir(self._root_path):
-            if f not in ['index', 'checksums', 'queue', 'index_updated']:
-                uids.extend(os.listdir(os.path.join(self._root_path, f)))
+            if os.path.isdir(os.path.join(self._root_path, f)) and len(f) == 2:
+                for g in os.listdir(os.path.join(self._root_path, f)):
+                    if len(g) == 36:
+                        uids.append(g)
         return uids
 
 _instance = None
diff --git a/src/olpc/datastore/metadatastore.py b/src/olpc/datastore/metadatastore.py
index 4f663a7..be3aad6 100644
--- a/src/olpc/datastore/metadatastore.py
+++ b/src/olpc/datastore/metadatastore.py
@@ -1,6 +1,4 @@
 import os
-import logging
-import errno
 
 from olpc.datastore import layoutmanager
 from olpc.datastore import metadatareader
diff --git a/src/olpc/datastore/migration.py b/src/olpc/datastore/migration.py
new file mode 100644
index 0000000..8b3a771
--- /dev/null
+++ b/src/olpc/datastore/migration.py
@@ -0,0 +1,54 @@
+import os
+import logging
+import traceback
+import sys
+
+import cjson
+
+from olpc.datastore import layoutmanager
+
+def migrate_from_0():
+    logging.info('Migrating datastore from version 0 to version 1')
+    root_path = layoutmanager.get_instance().get_root_path()
+    old_root_path = os.path.join(root_path, 'store')
+    for f in os.listdir(old_root_path):
+        uid, ext = os.path.splitext(f)
+        if ext != '.metadata':
+            continue
+
+        logging.info('Migrating entry %r' % uid)
+        try:
+            _migrate_metadata(root_path, old_root_path, uid)
+            _migrate_file(root_path, old_root_path, uid)
+            _migrate_preview(root_path, old_root_path, uid)
+        except Exception:
+            logging.warning('Failed to migrate entry %r:%s\n' %(uid, 
+                ''.join(traceback.format_exception(*sys.exc_info()))))
+
+    logging.info('Migration finished')
+
+def _migrate_metadata(root_path, old_root_path, uid):
+    dir_path = layoutmanager.get_instance().get_entry_path(uid)
+    metadata_path = os.path.join(dir_path, 'metadata')
+    os.makedirs(metadata_path)
+
+    old_metadata_path = os.path.join(old_root_path, uid + '.metadata')
+    metadata = cjson.decode(open(old_metadata_path, 'r').read())
+    for key, value in metadata.items():
+        f = open(os.path.join(metadata_path, key), 'w')
+        try:
+            f.write(str(value))
+        finally:
+            f.close()
+
+def _migrate_file(root_path, old_root_path, uid):
+    if os.path.exists(os.path.join(old_root_path, uid)):
+        dir_path = layoutmanager.get_instance().get_entry_path(uid)
+        os.rename(os.path.join(old_root_path, uid), os.path.join(dir_path, uid))
+
+def _migrate_preview(root_path, old_root_path, uid):
+    dir_path = layoutmanager.get_instance().get_entry_path(uid)
+    metadata_path = os.path.join(dir_path, 'metadata')
+    os.rename(os.path.join(old_root_path, 'preview', uid),
+              os.path.join(metadata_path, 'preview'))
+