Convert Journal entries that have been saved to a storage device in 0.82 #9658

This code will convert the old Journal entries to the new
scheme described above. When a device containing a folder .olpc.store
is plugged into the machine the conversion will happen
automatically in the background. Files that have the metadata converted
already from a previous run will be omitted from the conversion.
The .olpc.store won't be removed after completion to be able
to restore if anything goes wrong during conversion.

Note: Entries that do not contain a data file can not
be copied to a storage device anymore in 0.84
(this was possible in 0.82), for code simplification
we do not restore those entries from 0.82.

The conversion code is written defensive. If an entry can
not be converted it will be skipped, if the database is
broken the conversion is skipped etc. So the deletion of a data
file on the device or any corruption to the .olpc.store
should not cause a crash to Sugar.

2 files changed, 136 insertions, 13 deletions

diff --git a/src/jarabe/journal/model.py b/src/jarabe/journal/model.py
index 4060221..ffd3ad2 100644
--- a/src/jarabe/journal/model.py
+++ b/src/jarabe/journal/model.py
@@ -45,6 +45,8 @@ PROPERTIES = ['uid', 'title', 'mtime', 'timestamp', 'keep', 'buddies',
 
 PAGES_TO_CACHE = 5
 
+JOURNAL_METADATA_DIR = '.Sugar-Metadata'
+
 class _Cache(object):
 
     __gtype_name__ = 'model_Cache'
@@ -363,7 +365,8 @@ class InplaceResultSet(BaseResultSet):
                             add_to_list = False
 
                     if add_to_list:
-                        file_info = (full_path, stat, int(stat.st_mtime), metadata)
+                        file_info = (full_path, stat, int(stat.st_mtime),
+                                     metadata)
                         self._file_list.append(file_info)
 
                     self.progress.send(self)
@@ -396,7 +399,7 @@ def _get_file_metadata(path, stat):
             'mime_type': gio.content_type_guess(filename=path),
             'activity': '',
             'activity_id': '',
-            'icon-color': client.get_string('/desktop/sugar/user/color'),
+            'icon-color': '',
             'description': path}
 
 def _get_file_metadata_from_json(dir_path, filename, preview=False):
@@ -405,8 +408,8 @@ def _get_file_metadata_from_json(dir_path, filename, preview=False):
 
     """
     metadata = None
-    metadata_path = os.path.join(dir_path,
-                          '.' + filename + '.metadata')
+    metadata_path = os.path.join(dir_path, JOURNAL_METADATA_DIR,
+                                 filename + '.metadata')
     if os.path.exists(metadata_path):
         try:
             metadata = json.load(open(metadata_path))
@@ -416,8 +419,8 @@ def _get_file_metadata_from_json(dir_path, filename, preview=False):
         else:
             metadata['uid'] = os.path.join(dir_path, filename)
     if preview:
-        preview_path = os.path.join(dir_path,
-                                    '.' + filename + '.preview')
+        preview_path = os.path.join(dir_path, JOURNAL_METADATA_DIR,
+                                    filename + '.preview')
         if os.path.exists(preview_path):
             try:
                 metadata['preview'] = dbus.ByteArray(open(preview_path).read())
@@ -517,8 +520,10 @@ def delete(object_id):
         os.unlink(object_id)
         dir_path = os.path.dirname(object_id)
         filename = os.path.basename(object_id)
-        old_files = [os.path.join(dir_path, '.' + filename + '.metadata'),
-                     os.path.join(dir_path, '.' + filename + '.preview')]
+        old_files = [os.path.join(dir_path, JOURNAL_METADATA_DIR,
+                                  filename + '.metadata'),
+                     os.path.join(dir_path, JOURNAL_METADATA_DIR,
+                                  filename + '.preview')]
         for old_file in old_files:
             if os.path.exists(old_file):
                 try:
@@ -598,10 +603,16 @@ def _write_entry_on_external_device(metadata, file_path):
     if 'uid' in metadata_copy:
         del metadata_copy['uid']
 
+    metadata_dir_path = os.path.join(metadata['mountpoint'],
+                                     JOURNAL_METADATA_DIR)
+    if not os.path.exists(metadata_dir_path):
+        os.mkdir(metadata_dir_path)
+
     if 'preview' in metadata_copy:
         preview = metadata_copy['preview']
-        preview_fname = '.' + file_name + '.preview'
-        preview_path = os.path.join(metadata['mountpoint'], preview_fname)
+        preview_fname = file_name + '.preview'
+        preview_path = os.path.join(metadata['mountpoint'],
+                                    JOURNAL_METADATA_DIR, preview_fname)
         metadata_copy['preview'] = preview_fname
 
         (fh, fn) = tempfile.mkstemp(dir=metadata['mountpoint'])
@@ -610,7 +621,8 @@ def _write_entry_on_external_device(metadata, file_path):
         os.rename(fn, preview_path)
 
     metadata_path = os.path.join(metadata['mountpoint'],
-                                 '.' + file_name + '.metadata')
+                                 JOURNAL_METADATA_DIR,
+                                 file_name + '.metadata')
     (fh, fn) = tempfile.mkstemp(dir=metadata['mountpoint'])
     os.write(fh, json.dumps(metadata_copy))
     os.close(fh)
@@ -622,9 +634,11 @@ def _write_entry_on_external_device(metadata, file_path):
             os.rename(file_path, destination_path)
             old_fname = os.path.basename(file_path)
             old_files = [os.path.join(metadata['mountpoint'],
-                                      '.' + old_fname + '.metadata'),
+                                      JOURNAL_METADATA_DIR,
+                                      old_fname + '.metadata'),
                          os.path.join(metadata['mountpoint'],
-                                      '.' + old_fname + '.preview')]
+                                      JOURNAL_METADATA_DIR,
+                                      old_fname + '.preview')]
             for ofile in old_files:
                 if os.path.exists(ofile):
                     try:
diff --git a/src/jarabe/journal/volumestoolbar.py b/src/jarabe/journal/volumestoolbar.py
index 978028c..99f1725 100644
--- a/src/jarabe/journal/volumestoolbar.py
+++ b/src/jarabe/journal/volumestoolbar.py
@@ -17,6 +17,11 @@
 import logging
 import os
 from gettext import gettext as _
+import cPickle
+import xapian
+import json
+import tempfile
+import shutil
 
 import gobject
 import gio
@@ -30,6 +35,105 @@ from sugar.graphics.xocolor import XoColor
 from jarabe.journal import model
 from jarabe.view.palettes import VolumePalette
 
+_JOURNAL_0_METADATA_DIR = '.olpc.store'
+
+def _get_id(document):
+    """Get the ID for the document in the xapian database."""
+    tl = document.termlist()
+    try:
+        term = tl.skip_to('Q').term
+        if len(term) == 0 or term[0] != 'Q':
+            return None
+        return term[1:]
+    except StopIteration:
+        return None
+
+def _convert_entries(root):
+    """Converts the entries written by the datastore version 0.
+    The metadata and the preview will be written using the new
+    scheme for writing Journal entries to removable storage
+    devices.
+
+    - entries that do not have an associated file are not
+    converted.
+    - when done we write the file converted to the old metadat
+    directory, that we do not convert several times
+
+    """
+    try:
+        database = xapian.Database(os.path.join(root, _JOURNAL_0_METADATA_DIR,
+                                                'index'))
+    except xapian.DatabaseError, e:
+        logging.error('Convert DS-0 Journal entry. Error reading db: %s',
+                      os.path.join(root, _JOURNAL_0_METADATA_DIR, 'index'))
+        return
+
+    metadata_dir_path = os.path.join(root, model.JOURNAL_METADATA_DIR)
+    if not os.path.exists(metadata_dir_path):
+        os.mkdir(metadata_dir_path)
+
+    for i in range(1, database.get_lastdocid() + 1):
+        try:
+            document = database.get_document(i)
+        except xapian.DocNotFoundError, e:
+            logging.debug('Convert DS-0 Journal entry. ' \
+                              'Error getting document %s: %s', i, e)
+            continue
+
+        try:
+            metadata_loaded = cPickle.loads(document.get_data())
+        except cPickle.PickleError, e:
+            logging.debug('Convert DS-0 Journal entry. ' \
+                              'Error converting metadata: %s', e)
+            continue
+
+        if 'activity_id' in metadata_loaded and \
+                'mime_type' in metadata_loaded and \
+                'title' in metadata_loaded:
+            metadata = {}
+
+            uid = _get_id(document)
+            if uid is None:
+                continue
+
+            for key, value in metadata_loaded.items():
+                metadata[str(key)] = str(value[0])
+
+            if 'uid' not in metadata:
+                metadata['uid'] = uid
+
+            if 'filename' in metadata:
+                filename = metadata['filename']
+            else:
+                continue
+            if not os.path.exists(os.path.join(root, filename)):
+                continue
+
+            preview_path = os.path.join(root, _JOURNAL_0_METADATA_DIR,
+                                        'preview', uid)
+            if os.path.exists(preview_path):
+                preview_fname = filename + '.preview'
+                new_preview_path = os.path.join(root,
+                                                model.JOURNAL_METADATA_DIR,
+                                                preview_fname)
+                if not os.path.exists(new_preview_path):
+                    metadata['preview'] = preview_fname
+                    shutil.copy(preview_path, new_preview_path)
+
+            metadata_fname = filename + '.metadata'
+            metadata_path = os.path.join(root, model.JOURNAL_METADATA_DIR,
+                                         metadata_fname)
+            if not os.path.exists(metadata_path):
+                (fh, fn) = tempfile.mkstemp(dir=root)
+                os.write(fh, json.dumps(metadata))
+                os.close(fh)
+                os.rename(fn, metadata_path)
+
+            logging.debug('Convert DS-0 Journal entry. Entry converted: ' \
+                              'File=%s Metadata=%s',
+                          os.path.join(root, filename), metadata)
+
+
 class VolumesToolbar(gtk.Toolbar):
     __gtype_name__ = 'VolumesToolbar'
 
@@ -82,6 +186,11 @@ class VolumesToolbar(gtk.Toolbar):
     def _add_button(self, mount):
         logging.debug('VolumeToolbar._add_button: %r' % mount.get_name())
 
+        if os.path.exists(os.path.join(mount.get_root().get_path(),
+                                       _JOURNAL_0_METADATA_DIR)):
+            logging.debug('Convert DS-0 Journal entries.')
+            gobject.idle_add(_convert_entries, mount.get_root().get_path())
+
         button = VolumeButton(mount)
         button.props.group = self._volume_buttons[0]
         button.connect('toggled', self._button_toggled_cb)