Speed up source changes checking

4 files changed, 113 insertions, 36 deletions

diff --git a/.gitignore b/.gitignore
index fb6628a..0627f23 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 *.pyc
+*.so
 state/
 install/
 build/
diff --git a/Makefile b/Makefile
index 2a28311..5f90106 100644
--- a/Makefile
+++ b/Makefile
@@ -5,19 +5,26 @@ TIME=time -f "\n= Time =\n\nreal\t%e\nuser\t%U\nsys\t%S\n"
 
 all: build
 
+sourcestamp_cflags=`pkg-config --cflags python-2.7`
+sourcestamp_libs=`pkg-config --libs python-2.7`
+
+build-sourcestamp:
+	@gcc -shared -fPIC -o devbot/sourcestamp.so \
+		$(sourcestamp_cflags) $(sourcestamp_libs) devbot/sourcestamp.c
+
 check-system:
 	@$(COMMANDS_DIR)/check-system $(ARGS)
 
 pull:
 	@$(COMMANDS_DIR)/pull $(ARGS)
 
-build:
+build: build-sourcestamp
 	@$(TIME) $(COMMANDS_DIR)/build $(ARGS)
 
-run:
+run: build-sourcestamp
 	@$(COMMANDS_DIR)/run
 
-check:
+check: build-sourcestamp
 	@$(COMMANDS_DIR)/check
 
 shell:
@@ -29,9 +36,6 @@ bug-report:
 clean:
 	@$(COMMANDS_DIR)/clean
 
-distribute:
-	@$(COMMANDS_DIR)/distribute
-
 snapshot:
 	@$(COMMANDS_DIR)/snapshot
 
diff --git a/devbot/sourcestamp.c b/devbot/sourcestamp.c
new file mode 100644
index 0000000..ae6a8e8
--- /dev/null
+++ b/devbot/sourcestamp.c
@@ -0,0 +1,73 @@
+#include <Python.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <dirent.h>
+#include <string.h>
+#include <sys/stat.h>
+
+static PyObject *sourcestamp_compute(PyObject *self, PyObject *args);
+
+static PyMethodDef module_methods[] = {
+    {"compute", sourcestamp_compute, METH_VARARGS, NULL},
+    {NULL, NULL, 0, NULL}
+};
+
+PyMODINIT_FUNC
+initsourcestamp(void)
+{
+    Py_InitModule("sourcestamp", module_methods);
+}
+
+static void
+list_dir(const char *dir, time_t *mtime, int *n_files)
+{
+    DIR *dp;
+    struct dirent *entry;
+    struct stat statbuf;
+
+    dp = opendir(dir);
+
+    chdir(dir);
+
+    while ((entry = readdir(dp)) != NULL) {
+        *n_files = *n_files + 1;
+
+        lstat(entry->d_name, &statbuf);
+
+        if (statbuf.st_mtime > *mtime) {
+            *mtime = statbuf.st_mtime;
+        }
+
+        if(S_ISDIR(statbuf.st_mode)) {
+            if(strcmp(".", entry->d_name) == 0 ||
+               strcmp("..", entry->d_name) == 0 ||
+               strcmp(".git", entry->d_name) == 0)
+                continue;
+
+            list_dir(entry->d_name, mtime, n_files);
+        }
+    }
+
+    chdir("..");
+
+    closedir(dp);
+}
+
+static
+PyObject *sourcestamp_compute(PyObject *self, PyObject *args)
+{
+    time_t mtime = 0;
+    int n_files = 0;
+    const char *path;
+    char stamp[100];
+
+    if (!PyArg_ParseTuple(args, "s", &path)) {
+        return NULL;
+    }
+
+    list_dir(path, &mtime, &n_files);
+    snprintf(stamp, 100, "%d - %d", n_files, mtime);
+
+    return Py_BuildValue("s", stamp);
+}
diff --git a/devbot/state.py b/devbot/state.py
index 0a6f7b9..1feb4c2 100644
--- a/devbot/state.py
+++ b/devbot/state.py
@@ -1,53 +1,70 @@
-import hashlib
 import os
 import json
 
 from devbot import config
 
+try:
+    from devbot import sourcestamp
+    has_sourcestamp = True
+except ImportError:
+    has_sourcestamp = False
+
 _BUILT_MODULES = "builtmodules"
 _FULL_BUILD = "fullbuild"
 _SYSTEM_CHECK = "syscheck"
 
 
 def built_module_touch(module):
+    if not has_sourcestamp:
+        return
+
     built_modules = _load_state(_BUILT_MODULES, {})
 
-    source_hash = _compute_mtime_hash(module.get_source_dir())
-    built_modules[module.name] = {"source_hash": source_hash}
+    source_stamp = sourcestamp.compute(module.get_source_dir())
+    built_modules[module.name] = {"source_stamp": source_stamp}
 
     _save_state(_BUILT_MODULES, built_modules)
 
 
 def built_module_is_unchanged(module):
+    if not has_sourcestamp:
+        return False
+
     built_modules = _load_state(_BUILT_MODULES, {})
     if module.name not in built_modules:
         return False
 
     built_module = built_modules[module.name]
-    if "source_hash" not in built_module:
+    if "source_stamp" not in built_module:
         return False
 
-    old_source_hash = built_module["source_hash"]
-    new_source_hash = _compute_mtime_hash(module.get_source_dir())
+    old_source_stamp = built_module["source_stamp"]
+    new_source_stamp = sourcestamp.compute(module.get_source_dir())
 
-    return old_source_hash == new_source_hash
+    return old_source_stamp == new_source_stamp
 
 
 def system_check_is_unchanged():
+    if not has_sourcestamp:
+        return False
+
     system_check = _load_state(_SYSTEM_CHECK)
-    if not system_check or not "config_hash" in system_check:
+    if not system_check or not "config_stamp" in system_check:
         return False
 
-    config_hash = _compute_mtime_hash(config.config_dir)
+    config_stamp = sourcestamp.compute(config.config_dir)
 
-    return system_check["config_hash"] == config_hash
+    return system_check["config_stamp"] == config_stamp
 
 
 def system_check_touch():
+    if not has_sourcestamp:
+        return
+
     system_check = _load_state(_SYSTEM_CHECK, {})
 
-    config_hash = _compute_mtime_hash(config.config_dir)
-    system_check["config_hash"] = config_hash
+    config_stamp = sourcestamp.compute(config.config_dir)
+    system_check["config_stamp"] = config_stamp
 
     _save_state(_SYSTEM_CHECK, system_check)
 
@@ -100,21 +117,3 @@ def _save_state(name, state):
     with open(_get_state_path(name), "w+") as f:
         json.dump(state, f, indent=4)
         f.write('\n')
-
-
-def _compute_mtime_hash(path):
-    # For some reason if path is unicode we
-    # get a 10x slow down for some directories
-    path = str(path)
-
-    data = ""
-    for root, dirs, files in os.walk(path):
-        for name in files:
-            path = os.path.join(root, name)
-            mtime = os.lstat(path).st_mtime
-            data = "%s%s %s\n" % (data, mtime, path)
-
-            if ".git" in dirs:
-                dirs.remove(".git")
-
-    return hashlib.sha256(data).hexdigest()