From 98d76eaa0b806025809c76af65b82d735772e9a5 Mon Sep 17 00:00:00 2001 From: Daniel Narvaez Date: Tue, 23 Apr 2013 12:38:34 +0000 Subject: Speed up source changes checking --- diff --git a/.gitignore b/.gitignore index fb6628a..0627f23 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc +*.so state/ install/ build/ diff --git a/Makefile b/Makefile index 2a28311..5f90106 100644 --- a/Makefile +++ b/Makefile @@ -5,19 +5,26 @@ TIME=time -f "\n= Time =\n\nreal\t%e\nuser\t%U\nsys\t%S\n" all: build +sourcestamp_cflags=`pkg-config --cflags python-2.7` +sourcestamp_libs=`pkg-config --libs python-2.7` + +build-sourcestamp: + @gcc -shared -fPIC -o devbot/sourcestamp.so \ + $(sourcestamp_cflags) $(sourcestamp_libs) devbot/sourcestamp.c + check-system: @$(COMMANDS_DIR)/check-system $(ARGS) pull: @$(COMMANDS_DIR)/pull $(ARGS) -build: +build: build-sourcestamp @$(TIME) $(COMMANDS_DIR)/build $(ARGS) -run: +run: build-sourcestamp @$(COMMANDS_DIR)/run -check: +check: build-sourcestamp @$(COMMANDS_DIR)/check shell: @@ -29,9 +36,6 @@ bug-report: clean: @$(COMMANDS_DIR)/clean -distribute: - @$(COMMANDS_DIR)/distribute - snapshot: @$(COMMANDS_DIR)/snapshot diff --git a/devbot/sourcestamp.c b/devbot/sourcestamp.c new file mode 100644 index 0000000..ae6a8e8 --- /dev/null +++ b/devbot/sourcestamp.c @@ -0,0 +1,73 @@ +#include +#include +#include +#include +#include +#include +#include + +static PyObject *sourcestamp_compute(PyObject *self, PyObject *args); + +static PyMethodDef module_methods[] = { + {"compute", sourcestamp_compute, METH_VARARGS, NULL}, + {NULL, NULL, 0, NULL} +}; + +PyMODINIT_FUNC +initsourcestamp(void) +{ + Py_InitModule("sourcestamp", module_methods); +} + +static void +list_dir(const char *dir, time_t *mtime, int *n_files) +{ + DIR *dp; + struct dirent *entry; + struct stat statbuf; + + dp = opendir(dir); + + chdir(dir); + + while ((entry = readdir(dp)) != NULL) { + *n_files = *n_files + 1; + + lstat(entry->d_name, &statbuf); + + if (statbuf.st_mtime > *mtime) { + *mtime = statbuf.st_mtime; + } + + if(S_ISDIR(statbuf.st_mode)) { + if(strcmp(".", entry->d_name) == 0 || + strcmp("..", entry->d_name) == 0 || + strcmp(".git", entry->d_name) == 0) + continue; + + list_dir(entry->d_name, mtime, n_files); + } + } + + chdir(".."); + + closedir(dp); +} + +static +PyObject *sourcestamp_compute(PyObject *self, PyObject *args) +{ + time_t mtime = 0; + int n_files = 0; + const char *path; + char stamp[100]; + + if (!PyArg_ParseTuple(args, "s", &path)) { + return NULL; + } + + list_dir(path, &mtime, &n_files); + snprintf(stamp, 100, "%d - %d", n_files, mtime); + + return Py_BuildValue("s", stamp); +} diff --git a/devbot/state.py b/devbot/state.py index 0a6f7b9..1feb4c2 100644 --- a/devbot/state.py +++ b/devbot/state.py @@ -1,53 +1,70 @@ -import hashlib import os import json from devbot import config +try: + from devbot import sourcestamp + has_sourcestamp = True +except ImportError: + has_sourcestamp = False + _BUILT_MODULES = "builtmodules" _FULL_BUILD = "fullbuild" _SYSTEM_CHECK = "syscheck" def built_module_touch(module): + if not has_sourcestamp: + return + built_modules = _load_state(_BUILT_MODULES, {}) - source_hash = _compute_mtime_hash(module.get_source_dir()) - built_modules[module.name] = {"source_hash": source_hash} + source_stamp = sourcestamp.compute(module.get_source_dir()) + built_modules[module.name] = {"source_stamp": source_stamp} _save_state(_BUILT_MODULES, built_modules) def built_module_is_unchanged(module): + if not has_sourcestamp: + return False + built_modules = _load_state(_BUILT_MODULES, {}) if module.name not in built_modules: return False built_module = built_modules[module.name] - if "source_hash" not in built_module: + if "source_stamp" not in built_module: return False - old_source_hash = built_module["source_hash"] - new_source_hash = _compute_mtime_hash(module.get_source_dir()) + old_source_stamp = built_module["source_stamp"] + new_source_stamp = sourcestamp.compute(module.get_source_dir()) - return old_source_hash == new_source_hash + return old_source_stamp == new_source_stamp def system_check_is_unchanged(): + if not has_sourcestamp: + return False + system_check = _load_state(_SYSTEM_CHECK) - if not system_check or not "config_hash" in system_check: + if not system_check or not "config_stamp" in system_check: return False - config_hash = _compute_mtime_hash(config.config_dir) + config_stamp = sourcestamp.compute(config.config_dir) - return system_check["config_hash"] == config_hash + return system_check["config_stamp"] == config_stamp def system_check_touch(): + if not has_sourcestamp: + return + system_check = _load_state(_SYSTEM_CHECK, {}) - config_hash = _compute_mtime_hash(config.config_dir) - system_check["config_hash"] = config_hash + config_stamp = sourcestamp.compute(config.config_dir) + system_check["config_stamp"] = config_stamp _save_state(_SYSTEM_CHECK, system_check) @@ -100,21 +117,3 @@ def _save_state(name, state): with open(_get_state_path(name), "w+") as f: json.dump(state, f, indent=4) f.write('\n') - - -def _compute_mtime_hash(path): - # For some reason if path is unicode we - # get a 10x slow down for some directories - path = str(path) - - data = "" - for root, dirs, files in os.walk(path): - for name in files: - path = os.path.join(root, name) - mtime = os.lstat(path).st_mtime - data = "%s%s %s\n" % (data, mtime, path) - - if ".git" in dirs: - dirs.remove(".git") - - return hashlib.sha256(data).hexdigest() -- cgit v0.9.1