diff options
| -rw-r--r-- | Makefile | 2 | ||||
| -rw-r--r-- | Makefile.commands | 2 | ||||
| -rw-r--r-- | Makefile.config | 6 | ||||
| -rwxr-xr-x | commands/build | 7 | ||||
| -rwxr-xr-x | commands/check-system | 13 | ||||
| -rw-r--r-- | commands/common.py | 3 | ||||
| -rw-r--r-- | config/config.json | 3 | ||||
| -rw-r--r-- | config/modules/sugar.json | 4 | ||||
| -rw-r--r-- | devbot/build.py | 33 | ||||
| -rw-r--r-- | devbot/config.py | 23 | ||||
| -rw-r--r-- | devbot/git.py | 33 | ||||
| -rw-r--r-- | devbot/state.py | 104 | ||||
| -rw-r--r-- | devbot/system.py | 13 | ||||
| -rw-r--r-- | devbot/utils.py | 17 |
14 files changed, 158 insertions, 105 deletions
@@ -7,7 +7,7 @@ BASE_DIR=$(CURDIR) .PHONY: all -all: check-system pull build +all: pull build include Makefile.config include Makefile.commands diff --git a/Makefile.commands b/Makefile.commands index be82830..17908dd 100644 --- a/Makefile.commands +++ b/Makefile.commands @@ -10,7 +10,7 @@ pull: @$(COMMANDS_DIR)/pull build: - @$(COMMANDS_DIR)/build + @$(COMMANDS_DIR)/build $(ARGS) run: @$(COMMANDS_DIR)/run diff --git a/Makefile.config b/Makefile.config index 95883c1..5959c60 100644 --- a/Makefile.config +++ b/Makefile.config @@ -19,11 +19,13 @@ CONFIG_MODULES= \ CONFIG_PACKAGES_DIR=config/packages -OONFIG_PACKAGES= \ +CONFIG_PACKAGES= \ $(CONFIG_PACKAGES_DIR)/basesystem.json \ $(CONFIG_PACKAGES_DIR)/buildslave.json \ $(CONFIG_PACKAGES_DIR)/deps.json +CONFIG_MAIN=config/config.json + config-normalize: - @$(JSON_NORMALIZE) $(CONFIG_MODULES) $(OONFIG_PACKAGES) + @$(JSON_NORMALIZE) $(CONFIG_MODULES) $(CONFIG_PACKAGES) $(CONFIG_MAIN) @$(JSON_NORMALIZE) --sort-by name $(CONFIG_DEPS) diff --git a/commands/build b/commands/build index 6911394..714e783 100755 --- a/commands/build +++ b/commands/build @@ -5,18 +5,23 @@ import sys import common +from devbot import system from devbot import build parser = argparse.ArgumentParser() parser.add_argument("module", nargs="?", help="name of the module to build") +parser.add_argument("--full", action="store_true", help="force a full build") args = parser.parse_args() common.setup() +if not system.check(skip_if_unchanged=True): + sys.exit(1) + if args.module: success = build.build_one(args.module) else: - success = build.build() + success = build.build(full=args.full) if not success: sys.exit(1) diff --git a/commands/check-system b/commands/check-system index 1bfb3e6..f610bcb 100755 --- a/commands/check-system +++ b/commands/check-system @@ -1,6 +1,7 @@ #!/usr/bin/python -u import argparse +import sys import os import common @@ -16,14 +17,12 @@ parser.add_argument("--remove", action="store_true", help="remove all the unnecessary packages") parser.add_argument("--test", action="store_true", help="don't add or remove packages, test only") -parser.add_argument("--skip-if-unchanged", action="store_true", - help="skip if unchanged from the last check") args = parser.parse_args() interactive = "SUGAR_BUILDBOT" not in os.environ -system.check(update=args.update, - remove=args.remove, - test=args.test, - interactive=interactive, - skip_if_unchanged=args.skip_if_unchanged) +if not system.check(update=args.update, + remove=args.remove, + test=args.test, + interactive=interactive): + sys.exit(1) diff --git a/commands/common.py b/commands/common.py index 80c42b4..61d9537 100644 --- a/commands/common.py +++ b/commands/common.py @@ -7,8 +7,11 @@ sys.path.append(base_dir) from devbot import config from devbot import command +from devbot import git def setup(): + git.set_root_path(base_dir) + args = {"config_dir": os.path.join(base_dir, "config"), "install_dir": os.path.join(base_dir, "install"), "source_dir": os.path.join(base_dir, "source"), diff --git a/config/config.json b/config/config.json new file mode 100644 index 0000000..126e679 --- /dev/null +++ b/config/config.json @@ -0,0 +1,3 @@ +{ + "full_build": "1" +} diff --git a/config/modules/sugar.json b/config/modules/sugar.json index 20a54c3..d3585c1 100644 --- a/config/modules/sugar.json +++ b/config/modules/sugar.json @@ -43,8 +43,8 @@ "repo": "git://git.sugarlabs.org/gst-plugins-espeak/mainline.git" }, { + "distribute": true, "name": "sugar-runner", - "repo": "git://git.sugarlabs.org/sugar-runner/sugar-runner.git", - "distribute": true + "repo": "git://git.sugarlabs.org/sugar-runner/sugar-runner.git" } ] diff --git a/devbot/build.py b/devbot/build.py index 9823310..1e33c70 100644 --- a/devbot/build.py +++ b/devbot/build.py @@ -43,35 +43,20 @@ def pull(): return True -def build(): +def build(full=False): environ.setup() _ccache_reset() - modules = config.load_modules() - skipped = [] + if full or state.full_build_is_required(): + clean() - for module in modules[:]: - new_commit_id = module.get_commit_id() - if new_commit_id is None: - break + state.full_build_touch() - old_commit_id = state.get_built_commit_id(module) - if old_commit_id == new_commit_id: - modules.pop(0) - skipped.append(module.name) - else: - break - - if skipped: - print "\n* Skipping unchanged modules *\n" - print "\n".join(skipped) - - for module in modules: - state.remove_built_commit_id(module) - - for module in modules: - if not _build_module(module, config.get_log_path("build")): + for module in config.load_modules(): + if state.built_module_is_unchanged(module): + print "\n* Skipping unchanged module %s *" % module.name + elif not _build_module(module, config.get_log_path("build")): return False _ccache_print_stats() @@ -226,7 +211,7 @@ def _build_module(module, log=None): except subprocess.CalledProcessError: return False - state.touch_built_commit_id(module) + state.built_module_touch(module) return True diff --git a/devbot/config.py b/devbot/config.py index 93c82f7..3a396cd 100644 --- a/devbot/config.py +++ b/devbot/config.py @@ -5,7 +5,6 @@ import pkgutil import tempfile from devbot import distro -from devbot import utils from devbot import plugins from devbot import git @@ -51,12 +50,6 @@ class Module: def get_build_dir(self): return os.path.join(get_build_dir(), self.name) - def get_commit_id(self, tag="HEAD"): - if not os.path.exists(self.get_source_dir()): - return None - - return utils.get_commit_id(self.get_source_dir()) - def get_git_module(self): return git.Module(path=get_source_dir(), name=self.name, remote=self.repo, branch=self.branch, tag=self.tag, @@ -77,13 +70,6 @@ def _ensure_dir(dir): if not os.path.exists(dir): os.mkdir(dir) -def get_commit_id(): - commit_id = utils.get_commit_id(config_dir) - if commit_id is None: - commit_id = "snapshot" - - return commit_id - def setup(**kwargs): _load_plugins() @@ -261,7 +247,14 @@ def _read_index(dir_name, extra=[]): with open(os.path.join(index_dir, "index.json")) as f: files.extend(json.load(f)) return [os.path.join(index_dir, json_file) for json_file in files] - + +def get_full_build(): + config = None + with open(os.path.join(config_dir, "config.json")) as f: + config = json.load(f) + + return config["full_build"] + def load_packages(): packages = {} diff --git a/devbot/git.py b/devbot/git.py index 1e361f0..4a86bd6 100644 --- a/devbot/git.py +++ b/devbot/git.py @@ -2,6 +2,9 @@ import os import subprocess from devbot import command +from devbot import utils + +_root_path = None def _chdir(func): def wrapped(*args, **kwargs): @@ -72,6 +75,21 @@ class Module: return subprocess.check_output(["git", "describe"]).strip() @_chdir + def diff(self): + return subprocess.check_output(["git", "diff"]) + + @_chdir + def is_valid(self): + result = subprocess.call(["git", "rev-parse", "HEAD"], + stdout=utils.devnull, + stderr=utils.devnull) + return result == 0 + + @_chdir + def get_commit_id(self): + return subprocess.check_output(["git", "rev-parse", "HEAD"]).strip() + + @_chdir def get_annotation(self, tag): # FIXME this is fragile, there must be a better way @@ -101,3 +119,18 @@ class Module: command.run(["git", "clean", "-fdx"]) return True + +def set_root_path(path): + global _root_path + _root_path = path + +def get_root_module(): + remote = "git://git.sugarlabs.org/sugar-build/sugar-build.git" + + module = Module(name=os.path.basename(_root_path), + remote=remote, + path=os.path.dirname(_root_path)) + if not module.is_valid(): + return None + + return module diff --git a/devbot/state.py b/devbot/state.py index 258c594..b44652f 100644 --- a/devbot/state.py +++ b/devbot/state.py @@ -1,50 +1,93 @@ +import hashlib import os import json from devbot import config +from devbot import git -_state = None +_BUILT_MODULES = "builtmodules" +_FULL_BUILD = "fullbuild" +_SYSTEM_CHECK = "syscheck" -def _get_state_path(): - return os.path.join(config.build_state_dir, "state.json") +def _get_state_path(name): + return os.path.join(config.build_state_dir, "%s.json" % name) -def _get_state(): - global _state +def _load_state(name, default=None): + state = default - if _state is not None: - return _state + try: + with open(_get_state_path(name)) as f: + state = json.load(f) + except IOError: + pass + + return state + +def _save_state(name, state): + with open(_get_state_path(name), "w+") as f: + json.dump(state, f, indent=4) + f.write('\n') + +def _get_diff_hash(git_module): + diff = git_module.diff().strip() + if diff: + return hashlib.sha256(diff).hexdigest() + else: + return None - state_path = _get_state_path() - if os.path.exists(state_path): - _state = json.load(open(state_path)) +def _get_root_commit_id(): + git_module = git.get_root_module() + if git_module: + commit_id = git_module.get_commit_id() else: - _state = { "built_modules": {} } + commit_id = "snapshot" + + return commit_id + +def built_module_touch(module): + git_module = module.get_git_module() + built_modules = _load_state(_BUILT_MODULES, {}) + + info = {"commit": git_module.get_commit_id(), + "diff_hash": _get_diff_hash(git_module)} + built_modules[module.name] = info + + _save_state(_BUILT_MODULES, built_modules) + +def built_module_is_unchanged(module): + git_module = module.get_git_module() + built_modules = _load_state(_BUILT_MODULES, {}) + if module.name not in built_modules: + return False - return _state + info = built_modules[module.name] -def _state_changed(): - json.dump(_state, open(_get_state_path(), "w+")) + return info["diff_hash"] == _get_diff_hash(git_module) and \ + info["commit"] == git_module.get_commit_id() -def touch_built_commit_id(module): - _get_state()["built_modules"][module.name] = module.get_commit_id() - _state_changed() +def system_check_is_unchanged(): + system_check = _load_state(_SYSTEM_CHECK) + if not system_check: + return False -def remove_built_commit_id(module): - state = _get_state() + return system_check["commit"] == _get_root_commit_id() - if module.name in state["built_modules"]: - del state["built_modules"][module.name] - _state_changed() +def system_check_touch(): + system_check = _load_state(_SYSTEM_CHECK, {}) + system_check["commit"] = _get_root_commit_id() + _save_state(_SYSTEM_CHECK, system_check) -def get_built_commit_id(module): - return _get_state()["built_modules"].get(module.name, None) +def full_build_is_required(): + full_build = _load_state(_FULL_BUILD) + if not full_build: + return True -def get_last_system_check(): - return _get_state().get("last_system_check", None) + return not (full_build["last"] == config.get_full_build()) -def touch_last_system_check(): - _get_state()["last_system_check"] = config.get_commit_id() - _state_changed() +def full_build_touch(): + full_build = _load_state(_FULL_BUILD, {}) + full_build["last"] = config.get_full_build() + _save_state(_FULL_BUILD, full_build) def clean(): _state = None @@ -52,6 +95,7 @@ def clean(): print "Deleting state" try: - os.unlink(_get_state_path()) + for name in _BUILT_MODULES, _SYSTEM_CHECK, _FULL_BUILD: + os.unlink(_get_state_path(name)) except OSError: pass diff --git a/devbot/system.py b/devbot/system.py index 85e2349..4b014a8 100644 --- a/devbot/system.py +++ b/devbot/system.py @@ -4,6 +4,7 @@ import subprocess import sys from devbot import config +from devbot import git from devbot import distro from devbot import command from devbot import state @@ -156,8 +157,8 @@ def remove_packages(package_manager, packages): def check(remove=False, update=False, test=False, interactive=True, skip_if_unchanged=False): if skip_if_unchanged: - if config.get_commit_id() == state.get_last_system_check(): - return + if state.system_check_is_unchanged(): + return True package_manager = \ distro.get_package_manager(test=test, interactive=interactive) @@ -168,12 +169,12 @@ def check(remove=False, update=False, test=False, interactive=True, checks = config.load_prerequisites() if not run_checks(package_manager, checks, packages): - sys.exit(1) + return False xvfb_proc, orig_display = xvfb.start() if not run_checks(package_manager, config.load_checks(), packages): - sys.exit(1) + return False xvfb.stop(xvfb_proc, orig_display) @@ -185,4 +186,6 @@ def check(remove=False, update=False, test=False, interactive=True, if remove: remove_packages(package_manager, packages) - state.touch_last_system_check() + state.system_check_touch() + + return True diff --git a/devbot/utils.py b/devbot/utils.py index 2864948..aed5de0 100644 --- a/devbot/utils.py +++ b/devbot/utils.py @@ -1,18 +1 @@ -import os -import subprocess - devnull = open("/dev/null", "w") - -def get_commit_id(dir): - orig_cwd = os.getcwd() - os.chdir(dir) - - try: - commit_id = subprocess.check_output(["git", "rev-parse", "HEAD"], - stderr=devnull).strip() - except subprocess.CalledProcessError: - commit_id = None - - os.chdir(orig_cwd) - - return commit_id |