Add support for additional, read-only data stores

Some users may have direct access to additional data stores in
gdatastore format. Allow the user to "register" them by placing links
inside the (new) "extra-stores" directory and use them as additional
sources.

Modification is only supported for the primary store. The additional
ones are used for reading only. In particular, trying to remove an
entry that's contained in an extra store will yield an error (no
shadowing).

The extra stores will need to use exactly the same on-disk format as
the primary one. No automatic migration is done.

1 files changed, 28 insertions, 12 deletions

diff --git a/gdatastore/datastore.py b/gdatastore/datastore.py
index 9350209..6dd2170 100644
--- a/gdatastore/datastore.py
+++ b/gdatastore/datastore.py
@@ -513,7 +513,18 @@ class InternalApi(object):
         self._max_versions = gconf_client.get_int(
             '/desktop/sugar/datastore/max_versions')
         logging.debug('max_versions=%r', self._max_versions)
-        self._index = Index(os.path.join(self._base_dir, 'index'))
+        data_stores = [{'name': '',
+                        'index_dir': os.path.join(self._base_dir, 'index'),
+                        'git_dir': os.path.join(self._base_dir, 'git')}]
+        extras_dir = os.path.join(self._base_dir, 'extra-stores')
+        if not os.path.isdir(extras_dir):
+            os.makedirs(extras_dir)
+        for extra_name in os.listdir(extras_dir):
+            extra_dir = os.path.realpath(os.path.join(extras_dir, extra_name))
+            data_stores.append({'name': extra_name,
+                                'index_dir': os.path.join(extra_dir, 'index'),
+                                'git_dir': os.path.join(extra_dir, 'git')})
+        self._index = Index(data_stores)
         self._migrate()
         self._check_reindex()
         logging.info('ready')
@@ -528,7 +539,7 @@ class InternalApi(object):
         logging.debug('change_metadata(%r, %r)', object_id, metadata)
         metadata['tree_id'], metadata['version_id'] = object_id
         if 'creation_time' not in metadata:
-            old_metadata = self._index.retrieve(object_id)
+            old_metadata = self._index.retrieve(object_id)['metadata']
             metadata['creation_time'] = old_metadata['creation_time']
 
         self._index.store(object_id, metadata)
@@ -543,18 +554,21 @@ class InternalApi(object):
     def get_data_path(self, (tree_id, version_id), sender=None):
         logging.debug('get_data_path((%r, %r), %r)', tree_id, version_id,
                       sender)
-        metadata = self._index.retrieve((tree_id, version_id))
+        entry = self._index.retrieve((tree_id, version_id))
+        metadata = entry['metadata']
+        git_dir = entry['data_store']['git_dir']
         ref_name = _format_ref(tree_id, version_id)
         top_level_entries = self._git_call('ls-tree',
-                                           [ref_name]).splitlines()
+                                           [ref_name],
+                                           git_dir=git_dir).splitlines()
         if len(top_level_entries) == 1 and \
            top_level_entries[0].endswith('\tdata'):
             blob_hash = top_level_entries[0].split('\t')[0].split(' ')[2]
             mime_type = metadata.get('mime_type', '')
-            return self._checkout_file(blob_hash,
+            return self._checkout_file(blob_hash, git_dir,
                                        suffix=_guess_extension(mime_type))
 
-        return self._checkout_dir(ref_name)
+        return self._checkout_dir(ref_name, git_dir)
 
     def find(self, query_dict, options, query_string=None):
         logging.debug('find(%r, %r, %r)', query_dict, options, query_string)
@@ -579,7 +593,7 @@ class InternalApi(object):
         return self._index.find_unique_values(name)
 
     def get_properties(self, object_id):
-        return self._index.retrieve(object_id)
+        return self._index.retrieve(object_id)['metadata']
 
     def save(self, tree_id, parent_id, metadata, path, delete_after, async_cb,
              async_err_cb, allow_new_parent=False):
@@ -676,15 +690,16 @@ class InternalApi(object):
         for entry in old_versions:
             self.delete((entry['tree_id'], entry['version_id']))
 
-    def _checkout_file(self, blob_hash, suffix=''):
+    def _checkout_file(self, blob_hash, git_dir, suffix=''):
         fd, file_name = tempfile.mkstemp(dir=self._checkouts_dir, suffix=suffix)
         try:
-            self._git_call('cat-file', ['blob', blob_hash], stdout_fd=fd)
+            self._git_call('cat-file', ['blob', blob_hash], git_dir=git_dir,
+                           stdout_fd=fd)
         finally:
             os.close(fd)
         return file_name
 
-    def _checkout_dir(self, ref_name):
+    def _checkout_dir(self, ref_name, git_dir):
         # FIXME
         return ''
 
@@ -758,7 +773,8 @@ class InternalApi(object):
         return str(uuid.uuid4())
 
     def _git_call(self, command, args=None, input=None, input_fd=None,
-                  stdout_fd=None, work_dir=None, index_path=None):
+                  stdout_fd=None, work_dir=None, index_path=None,
+                  git_dir=None):
         env = dict(self._git_env)
         if work_dir:
             env['GIT_WORK_TREE'] = work_dir
@@ -767,7 +783,7 @@ class InternalApi(object):
         logging.debug('calling git %s, env=%r', ['git', command] + (args or []), env)
         pipe = Popen(['git', command] + (args or []), stdin=input_fd or PIPE,
                      stdout=stdout_fd or PIPE, stderr=PIPE, close_fds=True,
-                     cwd=self._git_dir, env=env)
+                     cwd=git_dir or self._git_dir, env=env)
         stdout, stderr = pipe.communicate(input)
         if pipe.returncode:
             raise GitError(pipe.returncode, stderr)