use layoutmanager as much as possible, except for the entries below checksums/

6 files changed, 67 insertions, 103 deletions

diff --git a/src/carquinyol/filestore.py b/src/carquinyol/filestore.py
index edf06a6..d34e3b1 100644
--- a/src/carquinyol/filestore.py
+++ b/src/carquinyol/filestore.py
@@ -37,7 +37,7 @@ class FileStore(object):
         if not os.path.exists(dir_path):
             os.makedirs(dir_path)
 
-        destination_path = os.path.join(dir_path, 'data')
+        destination_path = layoutmanager.get_instance().get_data_path(tree_id, version_id)
         if file_path:
             if not os.path.isfile(file_path):
                 raise ValueError('No file at %r' % file_path)
@@ -81,7 +81,7 @@ class FileStore(object):
         
         """
         dir_path = layoutmanager.get_instance().get_entry_path(tree_id, version_id)
-        file_path = os.path.join(dir_path, 'data')
+        file_path = layoutmanager.get_instance().get_data_path(tree_id, version_id)
         if not os.path.exists(file_path):
             logging.debug('Entry (%r,%r) doesnt have any file' % (tree_id,version_id))
             return ''
@@ -105,21 +105,13 @@ class FileStore(object):
         elif extension:
             extension = '.' + extension
 
-        destination_path = os.path.join(destination_dir, "%s-%s%s" % (tree_id, version_id, extension))
-
-        attempt = 1
-        while os.path.exists(destination_path):
-            if attempt > 10:
-                fd_, destination_path = tempfile.mkstemp(prefix="%s-%s" % (tree_id,version_id),
-                                                         suffix=extension,
-                                                         dir=destination_dir)
-                del fd_
-                os.unlink(destination_path)
-                break
-            else:
-                file_name = '%s-%s_%s%s' % (tree_id, version_id, attempt, extension)
-                destination_path = os.path.join(destination_dir, file_name)
-                attempt += 1
+        base_name = "%s-%s" % (tree_id, version_id)
+        fd_, destination_path = tempfile.mkstemp(prefix=base_name,
+                                                 suffix=extension,
+                                                 dir=destination_dir)
+        del fd_
+        # FIXME
+        os.unlink(destination_path)
 
         # Try to hard link from the original file to the targetpath. This can
         # fail if the file is in a different filesystem. Do a symlink instead.
@@ -128,39 +120,33 @@ class FileStore(object):
         except OSError, e:
             if e.errno == errno.EXDEV:
                 os.symlink(file_path, destination_path)
-            else:
-                raise
 
-        # Try to make the original file readable. This can fail if the file is
-        # in a FAT filesystem.
-        try:
-            os.chmod(file_path, 0604)
-        except OSError, e:
-            if e.errno != errno.EPERM:
+                # Try to make the original file readable. This can fail if the file is
+                # in a FAT filesystem.
+                try:
+                    os.chmod(file_path, 0444)
+                except OSError, e:
+                    if e.errno != errno.EPERM:
+                        raise
+            else:
                 raise
 
         return destination_path
 
     def get_file_path(self, tree_id, version_id):
-        dir_path = layoutmanager.get_instance().get_entry_path(tree_id, version_id)
-        return os.path.join(dir_path, 'data')
+        return layoutmanager.get_instance().get_data_path(tree_id, version_id)
 
     def delete(self, tree_id, version_id):
         """Remove the file associated to a given entry.
         
         """
-        dir_path = layoutmanager.get_instance().get_entry_path(tree_id, version_id)
-        file_path = os.path.join(dir_path, 'data')
+        file_path = layoutmanager.get_instance().get_data_path(tree_id, version_id)
         if os.path.exists(file_path):
             os.remove(file_path)
 
     def hard_link_entry(self, new_tree_id, new_version_id, existing_tree_id, existing_version_id):
-        existing_file = os.path.join(
-                layoutmanager.get_instance().get_entry_path(existing_tree_id, existing_version_id),
-                'data')
-        new_file = os.path.join(
-                layoutmanager.get_instance().get_entry_path(new_tree_id, new_version_id),
-                'data')
+        existing_file = layoutmanager.get_instance().get_data_path(existing_tree_id, existing_version_id)
+        new_file = layoutmanager.get_instance().get_data_path(new_tree_id, new_version_id)
 
         logging.debug('removing %r' % new_file)
         os.remove(new_file)
diff --git a/src/carquinyol/layoutmanager.py b/src/carquinyol/layoutmanager.py
index abc8376..8be2a11 100644
--- a/src/carquinyol/layoutmanager.py
+++ b/src/carquinyol/layoutmanager.py
@@ -57,6 +57,14 @@ class LayoutManager(object):
         # os.path.join() is just too slow
         return '%s/%s/%s-%s' % (self._root_path, tree_id[:2], tree_id, version_id)
 
+    def get_data_path(self, tree_id, version_id) :
+        # os.path.join() is just too slow
+        return '%s/%s/%s-%s/data' % (self._root_path, tree_id[:2], tree_id, version_id)
+
+    def get_metadata_path(self, tree_id, version_id) :
+        # os.path.join() is just too slow
+        return '%s/%s/%s-%s/metadata' % (self._root_path, tree_id[:2], tree_id, version_id)
+
     def get_root_path(self):
         return self._root_path
 
diff --git a/src/carquinyol/metadatareader.c b/src/carquinyol/metadatareader.c
index 08be17e..511b94a 100644
--- a/src/carquinyol/metadatareader.c
+++ b/src/carquinyol/metadatareader.c
@@ -8,7 +8,7 @@
 static PyObject *byte_array_type = NULL;
 
 int
-add_property(char *metadata_path, char *property_name, PyObject *dict,
+add_property(const char *metadata_path, char *property_name, PyObject *dict,
              int must_exist)
 {
     int file_path_size;
@@ -125,7 +125,7 @@ cleanup:
 }
 
 static PyObject *
-read_from_properties_list (char *metadata_path, PyObject *properties)
+read_from_properties_list (const char *metadata_path, PyObject *properties)
 {
     PyObject *dict = PyDict_New();
 
@@ -148,7 +148,7 @@ cleanup:
 }
 
 static PyObject *
-read_all_properties (char *metadata_path)
+read_all_properties (const char *metadata_path)
 {
     PyObject *dict = PyDict_New();
 	DIR *dir_stream = NULL;
@@ -198,21 +198,11 @@ metadatareader_retrieve(PyObject *unused, PyObject *args)
 {
     PyObject *dict = NULL;
     PyObject *properties = NULL;
-    const char *dir_path = NULL;
-    char *metadata_path = NULL;
+    const char *metadata_path = NULL;
 
-    if (!PyArg_ParseTuple(args, "sO:retrieve", &dir_path, &properties))
+    if (!PyArg_ParseTuple(args, "sO:retrieve", &metadata_path, &properties))
         return NULL;
 
-    // Build path to the metadata directory
-    int metadata_path_size = strlen(dir_path) + 10;
-    metadata_path = PyMem_Malloc(metadata_path_size);
-	if (metadata_path == NULL) {
-        PyErr_NoMemory();
-        return NULL;
-	}
-    snprintf (metadata_path, metadata_path_size, "%s/%s", dir_path, "metadata");
-
     if ((properties != Py_None) && (PyList_Size(properties) > 0)) {
         dict = read_from_properties_list(metadata_path, properties);
     } else {
@@ -225,7 +215,7 @@ metadatareader_retrieve(PyObject *unused, PyObject *args)
 }
 
 static PyMethodDef metadatareader_functions[] = {
-    {"retrieve", metadatareader_retrieve, METH_VARARGS, PyDoc_STR("Read a dictionary from a file")},
+    {"retrieve", metadatareader_retrieve, METH_VARARGS, PyDoc_STR("Read a dictionary from a directory with a single file (containing the content) per key")},
     {NULL, NULL, 0, NULL}
 };
 
diff --git a/src/carquinyol/metadatastore.py b/src/carquinyol/metadatastore.py
index 0e062e4..bc7bb12 100644
--- a/src/carquinyol/metadatastore.py
+++ b/src/carquinyol/metadatastore.py
@@ -7,11 +7,7 @@ MAX_SIZE = 256
 
 class MetadataStore(object):
     def store(self, tree_id, version_id, metadata):
-        dir_path = layoutmanager.get_instance().get_entry_path(tree_id, version_id)
-        if not os.path.exists(dir_path):
-            os.makedirs(dir_path)
-
-        metadata_path = os.path.join(dir_path, 'metadata')
+        metadata_path = layoutmanager.get_instance().get_metadata_path(tree_id, version_id)
         if not os.path.exists(metadata_path):
             os.makedirs(metadata_path)
         else:
@@ -38,19 +34,17 @@ class MetadataStore(object):
                 f.close()
 
     def retrieve(self, tree_id, version_id, properties=None):
-        dir_path = layoutmanager.get_instance().get_entry_path(tree_id, version_id)
-        return metadatareader.retrieve(dir_path, properties)
+        metadata_path = layoutmanager.get_instance().get_metadata_path(tree_id, version_id)
+        return metadatareader.retrieve(metadata_path, properties)
 
     def delete(self, tree_id, version_id):
-        dir_path = layoutmanager.get_instance().get_entry_path(tree_id, version_id)
-        metadata_path = os.path.join(dir_path, 'metadata')
+        metadata_path = layoutmanager.get_instance().get_metadata_path(tree_id, version_id)
         for key in os.listdir(metadata_path):
             os.remove(os.path.join(metadata_path, key))
         os.rmdir(metadata_path)
 
     def get_property(self, tree_id, version_id, key):
-        dir_path = layoutmanager.get_instance().get_entry_path(tree_id, version_id)
-        metadata_path = os.path.join(dir_path, 'metadata')
+        metadata_path = layoutmanager.get_instance().get_metadata_path(tree_id, version_id)
         property_path = os.path.join(metadata_path, key)
         if os.path.exists(property_path):
             return open(property_path, 'r').read()
@@ -58,8 +52,7 @@ class MetadataStore(object):
             return None
 
     def set_property(self, tree_id, version_id, key, value):
-        dir_path = layoutmanager.get_instance().get_entry_path(tree_id, version_id)
-        metadata_path = os.path.join(dir_path, 'metadata')
+        metadata_path = layoutmanager.get_instance().get_metadata_path(tree_id, version_id)
         property_path = os.path.join(metadata_path, key)
         open(property_path, 'w').write(value)
 
diff --git a/src/carquinyol/migration.py b/src/carquinyol/migration.py
index 7cfa6e6..f66dc2f 100644
--- a/src/carquinyol/migration.py
+++ b/src/carquinyol/migration.py
@@ -26,10 +26,13 @@ import uuid
 
 import cjson
 
-from carquinyol import layoutmanager
+from carquinyol import layoutmanager, metadatastore
 
 DATE_FORMAT = '%Y-%m-%dT%H:%M:%S'
 
+mstore = metadatastore.MetadataStore()
+
+
 def migrate_from_0():
     logging.info('Migrating datastore from version 0 to version 2')
 
@@ -61,8 +64,7 @@ def migrate_from_0():
 
 def _migrate_metadata_0(root_path, old_root_path, tree_id, version_id):
     dir_path = layoutmanager.get_instance().get_entry_path(tree_id, version_id)
-    metadata_path = os.path.join(dir_path, 'metadata')
-    os.makedirs(metadata_path)
+    metadata_path = layoutmanager.get_instance().get_metadata_path(tree_id, version_id)
 
     old_metadata_path = os.path.join(old_root_path, tree_id + '.metadata')
     metadata = cjson.decode(open(old_metadata_path, 'r').read())
@@ -79,31 +81,22 @@ def _migrate_metadata_0(root_path, old_root_path, tree_id, version_id):
     else :
         metadata['ctime'] = os.stat(old_metadata_path).st_ctime
 
-    for key, value in metadata.items():
-        try:
-            f = open(os.path.join(metadata_path, key), 'w')
-            try:
-                if isinstance(value, unicode):
-                    value = value.encode('utf-8')
-                if not isinstance(value, basestring):
-                    value = str(value)
-                f.write(value)
-            finally:
-                f.close()
-        except Exception:
-            logging.error(
-                    'Error while migrating property %s of entry %s: %s\n' % \
-                    (key, tree_id, traceback.format_exc()))
+    try:
+        mstore.store(tree_id, version_id, metadata)
+
+    except:
+        logging.error(
+                'Error while migrating property entry %s: %s\n' % \
+                (tree_id, traceback.format_exc()))
 
 def _migrate_file_0(root_path, old_root_path, tree_id, version_id):
     if os.path.exists(os.path.join(old_root_path, tree_id)):
-        dir_path = layoutmanager.get_instance().get_entry_path(tree_id, version_id)
+        new_data_path = layoutmanager.get_instance().get_data_path(tree_id, version_id)
         os.rename(os.path.join(old_root_path, tree_id),
-                  os.path.join(dir_path, 'data'))
+                  new_data_path)
 
 def _migrate_preview_0(root_path, old_root_path, tree_id, version_id):
-    dir_path = layoutmanager.get_instance().get_entry_path(tree_id, version_id)
-    metadata_path = os.path.join(dir_path, 'metadata')
+    metadata_path = layoutmanager.get_instance().get_metadata_path(tree_id, version_id)
     os.rename(os.path.join(old_root_path, 'preview', tree_id),
               os.path.join(metadata_path, 'preview'))
 
@@ -112,7 +105,7 @@ def migrate_from_1():
     logging.info('Migrating datastore from version 1 to version 2')
 
     root_path = layoutmanager.get_instance().get_root_path()
-    checksum_path = os.path.join(root_path, "checksums")
+    checksum_path = layoutmanager.get_instance().get_checksums_dir()
 
     version_ids = {}
     for hash02 in os.listdir(root_path):
@@ -129,23 +122,16 @@ def migrate_from_1():
             version_ids[tree_id] = version_id
             try:
                 new_path = layoutmanager.get_instance().get_entry_path(tree_id, version_id)
+                new_metadata_path = layoutmanager.get_instance().get_metadata_path(tree_id, version_id)
                 os.rename(os.path.join(root_path, hash02, tree_id),
                           new_path)
-                file(os.path.join(new_path, "metadata", "version_id"), "w").write(version_id)
-                file(os.path.join(new_path, "metadata", "parent_id"), "w").close()
-                uid_path = os.path.join(new_path, "metadata", "uid")
-                tid_path = os.path.join(new_path, "metadata", "tree_id")
-                timestamp_path = os.path.join(new_path, "metadata", "timestamp")
-                ctime_path = os.path.join(new_path, "metadata", "ctim")
-                if os.path.exists(uid_path) :
-                    os.rename(uid_path, tid_path)
-                else :
-                    file(tid_path, "w").write(tree_id)
-
-                if os.path.exists(timestamp_path) :
-                    os.rename(timestamp_path, ctime_path)
-                else :
-                    file(ctime_path, "w").write(str(os.stat(new_path).st_ctime))
+                metadata = mstore.retrieve(tree_id, version_id)
+                metadata['tree_id'] = tree_id
+                metadata['version_id'] = version_id
+                metadata['parent_id'] = ''
+                metadata['ctime'] = metadata.get('timestamp', os.stat(new_path).st_ctime)
+                metadata.pop('uid')
+                mstore.store(tree_id, version_id, metadata)
 
             except Exception:
                 logging.error('Error while migrating entry %r: %s\n' % \
@@ -159,7 +145,7 @@ def migrate_from_1():
 
             try :
                 os.rename(os.path.join(entries_path, tree_id),
-                          os.path.join(entries_path, "%s-%s" % (tree_id,version_ids[tree_id])))
+                          layoutmanager.get_instance().get_entry_path(tree_id, version_ids[tree_id]))
 
             except Exception:
                 logging.error('Error while migrating checksum entry %r / %r: %s\n' % \
diff --git a/src/carquinyol/optimizer.py b/src/carquinyol/optimizer.py
index 7aed8fb..19849e6 100644
--- a/src/carquinyol/optimizer.py
+++ b/src/carquinyol/optimizer.py
@@ -24,6 +24,7 @@ import gobject
 
 from carquinyol import layoutmanager
 
+# TODO: use layoutmanager for entries in 'checksums' directory
 class Optimizer(object):
     """Optimizes disk space usage by detecting duplicates and sharing storage.
     """