From 69d077f81fd00cb9b462030ffb6b8cc8cd9b25c4 Mon Sep 17 00:00:00 2001 From: Daniel Narvaez Date: Tue, 25 Dec 2012 18:20:39 +0000 Subject: Merge branch 'testing' --- (limited to 'devbot') diff --git a/devbot/build.py b/devbot/build.py index 9823310..1e33c70 100644 --- a/devbot/build.py +++ b/devbot/build.py @@ -43,35 +43,20 @@ def pull(): return True -def build(): +def build(full=False): environ.setup() _ccache_reset() - modules = config.load_modules() - skipped = [] + if full or state.full_build_is_required(): + clean() - for module in modules[:]: - new_commit_id = module.get_commit_id() - if new_commit_id is None: - break + state.full_build_touch() - old_commit_id = state.get_built_commit_id(module) - if old_commit_id == new_commit_id: - modules.pop(0) - skipped.append(module.name) - else: - break - - if skipped: - print "\n* Skipping unchanged modules *\n" - print "\n".join(skipped) - - for module in modules: - state.remove_built_commit_id(module) - - for module in modules: - if not _build_module(module, config.get_log_path("build")): + for module in config.load_modules(): + if state.built_module_is_unchanged(module): + print "\n* Skipping unchanged module %s *" % module.name + elif not _build_module(module, config.get_log_path("build")): return False _ccache_print_stats() @@ -226,7 +211,7 @@ def _build_module(module, log=None): except subprocess.CalledProcessError: return False - state.touch_built_commit_id(module) + state.built_module_touch(module) return True diff --git a/devbot/config.py b/devbot/config.py index 93c82f7..3a396cd 100644 --- a/devbot/config.py +++ b/devbot/config.py @@ -5,7 +5,6 @@ import pkgutil import tempfile from devbot import distro -from devbot import utils from devbot import plugins from devbot import git @@ -51,12 +50,6 @@ class Module: def get_build_dir(self): return os.path.join(get_build_dir(), self.name) - def get_commit_id(self, tag="HEAD"): - if not os.path.exists(self.get_source_dir()): - return None - - return utils.get_commit_id(self.get_source_dir()) - def get_git_module(self): return git.Module(path=get_source_dir(), name=self.name, remote=self.repo, branch=self.branch, tag=self.tag, @@ -77,13 +70,6 @@ def _ensure_dir(dir): if not os.path.exists(dir): os.mkdir(dir) -def get_commit_id(): - commit_id = utils.get_commit_id(config_dir) - if commit_id is None: - commit_id = "snapshot" - - return commit_id - def setup(**kwargs): _load_plugins() @@ -261,7 +247,14 @@ def _read_index(dir_name, extra=[]): with open(os.path.join(index_dir, "index.json")) as f: files.extend(json.load(f)) return [os.path.join(index_dir, json_file) for json_file in files] - + +def get_full_build(): + config = None + with open(os.path.join(config_dir, "config.json")) as f: + config = json.load(f) + + return config["full_build"] + def load_packages(): packages = {} diff --git a/devbot/git.py b/devbot/git.py index 1e361f0..4a86bd6 100644 --- a/devbot/git.py +++ b/devbot/git.py @@ -2,6 +2,9 @@ import os import subprocess from devbot import command +from devbot import utils + +_root_path = None def _chdir(func): def wrapped(*args, **kwargs): @@ -72,6 +75,21 @@ class Module: return subprocess.check_output(["git", "describe"]).strip() @_chdir + def diff(self): + return subprocess.check_output(["git", "diff"]) + + @_chdir + def is_valid(self): + result = subprocess.call(["git", "rev-parse", "HEAD"], + stdout=utils.devnull, + stderr=utils.devnull) + return result == 0 + + @_chdir + def get_commit_id(self): + return subprocess.check_output(["git", "rev-parse", "HEAD"]).strip() + + @_chdir def get_annotation(self, tag): # FIXME this is fragile, there must be a better way @@ -101,3 +119,18 @@ class Module: command.run(["git", "clean", "-fdx"]) return True + +def set_root_path(path): + global _root_path + _root_path = path + +def get_root_module(): + remote = "git://git.sugarlabs.org/sugar-build/sugar-build.git" + + module = Module(name=os.path.basename(_root_path), + remote=remote, + path=os.path.dirname(_root_path)) + if not module.is_valid(): + return None + + return module diff --git a/devbot/state.py b/devbot/state.py index 258c594..b44652f 100644 --- a/devbot/state.py +++ b/devbot/state.py @@ -1,50 +1,93 @@ +import hashlib import os import json from devbot import config +from devbot import git -_state = None +_BUILT_MODULES = "builtmodules" +_FULL_BUILD = "fullbuild" +_SYSTEM_CHECK = "syscheck" -def _get_state_path(): - return os.path.join(config.build_state_dir, "state.json") +def _get_state_path(name): + return os.path.join(config.build_state_dir, "%s.json" % name) -def _get_state(): - global _state +def _load_state(name, default=None): + state = default - if _state is not None: - return _state + try: + with open(_get_state_path(name)) as f: + state = json.load(f) + except IOError: + pass + + return state + +def _save_state(name, state): + with open(_get_state_path(name), "w+") as f: + json.dump(state, f, indent=4) + f.write('\n') + +def _get_diff_hash(git_module): + diff = git_module.diff().strip() + if diff: + return hashlib.sha256(diff).hexdigest() + else: + return None - state_path = _get_state_path() - if os.path.exists(state_path): - _state = json.load(open(state_path)) +def _get_root_commit_id(): + git_module = git.get_root_module() + if git_module: + commit_id = git_module.get_commit_id() else: - _state = { "built_modules": {} } + commit_id = "snapshot" + + return commit_id + +def built_module_touch(module): + git_module = module.get_git_module() + built_modules = _load_state(_BUILT_MODULES, {}) + + info = {"commit": git_module.get_commit_id(), + "diff_hash": _get_diff_hash(git_module)} + built_modules[module.name] = info + + _save_state(_BUILT_MODULES, built_modules) + +def built_module_is_unchanged(module): + git_module = module.get_git_module() + built_modules = _load_state(_BUILT_MODULES, {}) + if module.name not in built_modules: + return False - return _state + info = built_modules[module.name] -def _state_changed(): - json.dump(_state, open(_get_state_path(), "w+")) + return info["diff_hash"] == _get_diff_hash(git_module) and \ + info["commit"] == git_module.get_commit_id() -def touch_built_commit_id(module): - _get_state()["built_modules"][module.name] = module.get_commit_id() - _state_changed() +def system_check_is_unchanged(): + system_check = _load_state(_SYSTEM_CHECK) + if not system_check: + return False -def remove_built_commit_id(module): - state = _get_state() + return system_check["commit"] == _get_root_commit_id() - if module.name in state["built_modules"]: - del state["built_modules"][module.name] - _state_changed() +def system_check_touch(): + system_check = _load_state(_SYSTEM_CHECK, {}) + system_check["commit"] = _get_root_commit_id() + _save_state(_SYSTEM_CHECK, system_check) -def get_built_commit_id(module): - return _get_state()["built_modules"].get(module.name, None) +def full_build_is_required(): + full_build = _load_state(_FULL_BUILD) + if not full_build: + return True -def get_last_system_check(): - return _get_state().get("last_system_check", None) + return not (full_build["last"] == config.get_full_build()) -def touch_last_system_check(): - _get_state()["last_system_check"] = config.get_commit_id() - _state_changed() +def full_build_touch(): + full_build = _load_state(_FULL_BUILD, {}) + full_build["last"] = config.get_full_build() + _save_state(_FULL_BUILD, full_build) def clean(): _state = None @@ -52,6 +95,7 @@ def clean(): print "Deleting state" try: - os.unlink(_get_state_path()) + for name in _BUILT_MODULES, _SYSTEM_CHECK, _FULL_BUILD: + os.unlink(_get_state_path(name)) except OSError: pass diff --git a/devbot/system.py b/devbot/system.py index 85e2349..4b014a8 100644 --- a/devbot/system.py +++ b/devbot/system.py @@ -4,6 +4,7 @@ import subprocess import sys from devbot import config +from devbot import git from devbot import distro from devbot import command from devbot import state @@ -156,8 +157,8 @@ def remove_packages(package_manager, packages): def check(remove=False, update=False, test=False, interactive=True, skip_if_unchanged=False): if skip_if_unchanged: - if config.get_commit_id() == state.get_last_system_check(): - return + if state.system_check_is_unchanged(): + return True package_manager = \ distro.get_package_manager(test=test, interactive=interactive) @@ -168,12 +169,12 @@ def check(remove=False, update=False, test=False, interactive=True, checks = config.load_prerequisites() if not run_checks(package_manager, checks, packages): - sys.exit(1) + return False xvfb_proc, orig_display = xvfb.start() if not run_checks(package_manager, config.load_checks(), packages): - sys.exit(1) + return False xvfb.stop(xvfb_proc, orig_display) @@ -185,4 +186,6 @@ def check(remove=False, update=False, test=False, interactive=True, if remove: remove_packages(package_manager, packages) - state.touch_last_system_check() + state.system_check_touch() + + return True diff --git a/devbot/utils.py b/devbot/utils.py index 2864948..aed5de0 100644 --- a/devbot/utils.py +++ b/devbot/utils.py @@ -1,18 +1 @@ -import os -import subprocess - devnull = open("/dev/null", "w") - -def get_commit_id(dir): - orig_cwd = os.getcwd() - os.chdir(dir) - - try: - commit_id = subprocess.check_output(["git", "rev-parse", "HEAD"], - stderr=devnull).strip() - except subprocess.CalledProcessError: - commit_id = None - - os.chdir(orig_cwd) - - return commit_id -- cgit v0.9.1