Modularize metadatareader.c and speed it up a bit more

1 files changed, 165 insertions, 126 deletions

diff --git a/src/olpc/datastore/metadatareader.c b/src/olpc/datastore/metadatareader.c
index ce6d38e..f0cec93 100644
--- a/src/olpc/datastore/metadatareader.c
+++ b/src/olpc/datastore/metadatareader.c
@@ -7,161 +7,103 @@
 
 static PyObject *byte_array_type = NULL;
 
-static PyObject *
-metadatareader_retrieve(PyObject *unused, PyObject *args)
+int
+add_property(char *metadata_path, char *property_name, PyObject *dict,
+             int must_exist)
 {
-    PyObject *dict = NULL;
-    PyObject *properties = NULL;
-    const char *dir_path = NULL;
-    char *metadata_path = NULL;
-	DIR *dir_stream = NULL;
-	struct dirent *dir_entry = NULL;
+    int file_path_size;
     char *file_path = NULL;
     FILE *file = NULL;
+    long file_size;
 	char *value_buf = NULL;
+    PyObject *value = NULL;
+    struct stat file_stat;
 
-    if (!PyArg_ParseTuple(args, "sO:retrieve", &dir_path, &properties))
-        return NULL;
-
-    // Build path to the metadata directory
-    int metadata_path_size = strlen(dir_path) + 10;
-    metadata_path = PyMem_Malloc(metadata_path_size);
-	if (metadata_path == NULL) {
+    // Build path of the property file
+    file_path_size = strlen(metadata_path) + 1 + strlen(property_name) + 1;
+    file_path = PyMem_Malloc(file_path_size);
+	if (file_path == NULL) {
         PyErr_NoMemory();
-        goto cleanup;
+        return 0;
 	}
-    snprintf (metadata_path, metadata_path_size, "%s/%s", dir_path, "metadata");
+    snprintf (file_path, file_path_size, "%s/%s", metadata_path, property_name);
 
-    dir_stream = opendir (metadata_path);
-	if (dir_stream == NULL) {
+    if ((!must_exist) && (stat(file_path, &file_stat) != 0)) {
+        PyMem_Free(file_path);
+        return;
+    }
+
+    file = fopen(file_path, "r");
+    if (file == NULL) {
 	    char buf[256];
-	    snprintf(buf, sizeof(buf), "Couldn't open metadata directory %s",
-	             metadata_path);
+	    snprintf(buf, sizeof(buf), "Cannot open property file %s: %s",
+	             file_path, strerror(errno));
         PyErr_SetString(PyExc_IOError, buf);
         goto cleanup;
-	}
-
-    dict = PyDict_New();
-
-	dir_entry = readdir(dir_stream);
-    while (dir_entry != NULL) {
-        long file_size;
-        int file_path_size;
-        PyObject *value = NULL;
+    }
 
-        // Skip . and ..
-        if (dir_entry->d_name[0] == '.' &&
-            (strlen(dir_entry->d_name) == 1 || 
-                (dir_entry->d_name[1] == '.' &&
-                    strlen(dir_entry->d_name) == 2)))
-			goto next_property;
+    // Get file size
+    fseek (file, 0, SEEK_END);
+    file_size = ftell (file);
+    rewind (file);
 
-        // Check if the property is in the properties list
-        if ((properties != Py_None) && (PyList_Size(properties) > 0)) {
-            int found = 0;
-            int i;
-            for (i = 0; i < PyList_Size(properties); i++) {
-                PyObject *property = PyList_GetItem(properties, i);
-                if (!strcmp (dir_entry->d_name, PyString_AsString (property))) {
-                    found = 1;
-                }
-            }
-            if (!found) {
-                goto next_property;
-            }
+    if (file_size == 0) {
+        // Empty property
+        value = PyString_FromString("");
+        if (value == NULL) {
+            PyErr_SetString(PyExc_ValueError,
+                            "Failed to convert value to python string");
+            goto cleanup;
+        }
+    } else {
+        if (file_size > MAX_PROPERTY_LENGTH) {
+            PyErr_SetString(PyExc_ValueError, "Property file too big");
+            goto cleanup;
         }
 
-        // Build path of the property file
-        file_path_size = strlen(metadata_path) + 1 + strlen(dir_entry->d_name) +
-                         1;
-        file_path = PyMem_Malloc(file_path_size);
-    	if (file_path == NULL) {
+        // Read the whole file
+    	value_buf = PyMem_Malloc(file_size);
+    	if (value_buf == NULL) {
             PyErr_NoMemory();
             goto cleanup;
     	}
-        snprintf (file_path, file_path_size, "%s/%s", metadata_path,
-                  dir_entry->d_name);
-
-        file = fopen(file_path, "r");
-        if (file == NULL) {
+        long read_size = fread(value_buf, 1, file_size, file);
+        if (read_size < file_size) {
     	    char buf[256];
-    	    snprintf(buf, sizeof(buf), "Cannot open property file %s: %s",
-    	             file_path, strerror(errno));
+    	    snprintf(buf, sizeof(buf),
+    	             "Error while reading property file %s", file_path);
             PyErr_SetString(PyExc_IOError, buf);
             goto cleanup;
         }
 
-        // Get file size
-        fseek (file, 0, SEEK_END);
-        file_size = ftell (file);
-        rewind (file);
-
-        if (file_size == 0) {
-            // Empty property
-            value = PyString_FromString("");
-            if (value == NULL) {
-                PyErr_SetString(PyExc_ValueError,
-                                "Failed to convert value to python string");
-                goto cleanup;
-            }
-        } else {
-            if (file_size > MAX_PROPERTY_LENGTH) {
-                PyErr_SetString(PyExc_ValueError, "Property file too big");
-                goto cleanup;
-            }
-
-            // Read the whole file
-        	value_buf = PyMem_Malloc(file_size);
-        	if (value_buf == NULL) {
-                PyErr_NoMemory();
-                goto cleanup;
-        	}
-            long read_size = fread(value_buf, 1, file_size, file);
-            if (read_size < file_size) {
-        	    char buf[256];
-        	    snprintf(buf, sizeof(buf),
-        	             "Error while reading property file %s", file_path);
-                PyErr_SetString(PyExc_IOError, buf);
-                goto cleanup;
-            }
-
-            // Convert value to dbus.ByteArray
-            PyObject *args = Py_BuildValue("(s#)", value_buf, file_size);
-            value = PyObject_CallObject(byte_array_type, args);
-            if (value == NULL) {
-                PyErr_SetString(PyExc_ValueError,
-                                "Failed to convert value to dbus.ByteArray");
-                goto cleanup;
-            }
-        }
+        fclose(file);
+        file = NULL;
+
+        // Convert value to dbus.ByteArray
+        PyObject *args = Py_BuildValue("(s#)", value_buf, file_size);
 
-        // Add property to the metadata dict
-        if (PyDict_SetItemString(dict, dir_entry->d_name, value) == -1) {
+        PyMem_Free(value_buf);
+        value_buf = NULL;
+
+        value = PyObject_CallObject(byte_array_type, args);
+        if (value == NULL) {
             PyErr_SetString(PyExc_ValueError,
-                            "Failed to add property to dictionary");
+                            "Failed to convert value to dbus.ByteArray");
             goto cleanup;
         }
+    }
 
-    next_property:
-        if (file_path) {
-            PyMem_Free(file_path);
-            file_path = NULL;
-        }
-        if (file) {
-            fclose(file);
-            file = NULL;
-        }
-        if (value_buf) {
-            PyMem_Free(value_buf);
-            value_buf = NULL;
-        }
-
-    	dir_entry = readdir(dir_stream);
+    // Add property to the metadata dict
+    if (PyDict_SetItemString(dict, property_name, value) == -1) {
+        PyErr_SetString(PyExc_ValueError,
+                        "Failed to add property to dictionary");
+        goto cleanup;
     }
 
-	closedir(dir_stream);
+    Py_DECREF(value);
+    PyMem_Free(file_path);
 
-    return dict;
+    return 1;
 
 cleanup:
     if (file_path) {
@@ -170,11 +112,77 @@ cleanup:
     if (value_buf) {
         PyMem_Free(value_buf);
     }
+    if (file) {
+        fclose(file);
+    }
+    if (value) {
+        Py_DECREF(value);
+    }
+    return 0;    
+}
+
+static PyObject *
+read_from_properties_list (char *metadata_path, PyObject *properties)
+{
+    PyObject *dict = PyDict_New();
+
+    int i;
+    for (i = 0; i < PyList_Size(properties); i++) {
+        PyObject *property = PyList_GetItem(properties, i);
+        char *property_name = PyString_AsString (property);
+
+        if (add_property(metadata_path, property_name, dict, 0) == 0)
+            goto cleanup;
+    }
+
+    return dict;
+
+cleanup:
     if (dict) {
         Py_DECREF(dict);
     }
-    if (file) {
-        fclose(file);
+    return NULL;        
+}
+
+static PyObject *
+read_all_properties (char *metadata_path)
+{
+    PyObject *dict = PyDict_New();
+	DIR *dir_stream = NULL;
+	struct dirent *dir_entry = NULL;
+
+    dir_stream = opendir (metadata_path);
+	if (dir_stream == NULL) {
+	    char buf[256];
+	    snprintf(buf, sizeof(buf), "Couldn't open metadata directory %s",
+	             metadata_path);
+        PyErr_SetString(PyExc_IOError, buf);
+        goto cleanup;
+	}
+
+	dir_entry = readdir(dir_stream);
+    while (dir_entry != NULL) {
+        // Skip . and ..
+        if (dir_entry->d_name[0] == '.' &&
+            (strlen(dir_entry->d_name) == 1 || 
+                (dir_entry->d_name[1] == '.' &&
+                    strlen(dir_entry->d_name) == 2)))
+			goto next_property;
+
+        if (add_property(metadata_path, dir_entry->d_name, dict, 1) == 0)
+            goto cleanup;
+
+    next_property:
+    	dir_entry = readdir(dir_stream);
+    }
+
+	closedir(dir_stream);
+
+    return dict;
+
+cleanup:
+    if (dict) {
+        Py_DECREF(dict);
     }
     if (dir_stream) {
     	closedir(dir_stream);
@@ -182,6 +190,37 @@ cleanup:
     return NULL;        
 }
 
+static PyObject *
+metadatareader_retrieve(PyObject *unused, PyObject *args)
+{
+    PyObject *dict = NULL;
+    PyObject *properties = NULL;
+    const char *dir_path = NULL;
+    char *metadata_path = NULL;
+
+    if (!PyArg_ParseTuple(args, "sO:retrieve", &dir_path, &properties))
+        return NULL;
+
+    // Build path to the metadata directory
+    int metadata_path_size = strlen(dir_path) + 10;
+    metadata_path = PyMem_Malloc(metadata_path_size);
+	if (metadata_path == NULL) {
+        PyErr_NoMemory();
+        return NULL;
+	}
+    snprintf (metadata_path, metadata_path_size, "%s/%s", dir_path, "metadata");
+
+    if ((properties != Py_None) && (PyList_Size(properties) > 0)) {
+        dict = read_from_properties_list(metadata_path, properties);
+    } else {
+        dict = read_all_properties(metadata_path);
+    }
+
+    PyMem_Free(metadata_path);
+
+    return dict;
+}
+
 static PyMethodDef metadatareader_functions[] = {
     {"retrieve", metadatareader_retrieve, METH_VARARGS, PyDoc_STR("Read a dictionary from a file")},
     {NULL, NULL, 0, NULL}