Merge branch 'testing'

6 files changed, 132 insertions, 91 deletions

diff --git a/devbot/build.py b/devbot/build.py
index 9823310..1e33c70 100644
--- a/devbot/build.py
+++ b/devbot/build.py
@@ -43,35 +43,20 @@ def pull():
 
     return True
 
-def build():
+def build(full=False):
     environ.setup()
 
     _ccache_reset()
 
-    modules = config.load_modules()
-    skipped = []
+    if full or state.full_build_is_required():
+        clean()
 
-    for module in modules[:]:
-        new_commit_id = module.get_commit_id()
-        if new_commit_id is None:
-            break
+    state.full_build_touch()
 
-        old_commit_id = state.get_built_commit_id(module)
-        if old_commit_id == new_commit_id:
-            modules.pop(0)
-            skipped.append(module.name)
-        else:
-            break
-
-    if skipped:
-        print "\n* Skipping unchanged modules *\n"
-        print "\n".join(skipped)
-
-    for module in modules:
-        state.remove_built_commit_id(module)
-
-    for module in modules:
-        if not _build_module(module, config.get_log_path("build")):
+    for module in config.load_modules():
+        if state.built_module_is_unchanged(module):
+            print "\n* Skipping unchanged module %s *" % module.name
+        elif not _build_module(module, config.get_log_path("build")):
             return False
 
     _ccache_print_stats()
@@ -226,7 +211,7 @@ def _build_module(module, log=None):
     except subprocess.CalledProcessError:
         return False
 
-    state.touch_built_commit_id(module)
+    state.built_module_touch(module)
 
     return True
 
diff --git a/devbot/config.py b/devbot/config.py
index 93c82f7..3a396cd 100644
--- a/devbot/config.py
+++ b/devbot/config.py
@@ -5,7 +5,6 @@ import pkgutil
 import tempfile
 
 from devbot import distro
-from devbot import utils
 from devbot import plugins
 from devbot import git
 
@@ -51,12 +50,6 @@ class Module:
     def get_build_dir(self):
         return os.path.join(get_build_dir(), self.name)
 
-    def get_commit_id(self, tag="HEAD"):
-        if not os.path.exists(self.get_source_dir()):
-            return None
-
-        return utils.get_commit_id(self.get_source_dir())
-
     def get_git_module(self):
         return git.Module(path=get_source_dir(), name=self.name,
                           remote=self.repo, branch=self.branch, tag=self.tag,
@@ -77,13 +70,6 @@ def _ensure_dir(dir):
     if not os.path.exists(dir):
         os.mkdir(dir)
 
-def get_commit_id():
-    commit_id = utils.get_commit_id(config_dir)
-    if commit_id is None:
-        commit_id = "snapshot"
-
-    return commit_id
-
 def setup(**kwargs):
     _load_plugins()
 
@@ -261,7 +247,14 @@ def _read_index(dir_name, extra=[]):
     with open(os.path.join(index_dir, "index.json")) as f:
         files.extend(json.load(f))
         return [os.path.join(index_dir, json_file) for json_file in files]
- 
+
+def get_full_build():
+    config = None
+    with open(os.path.join(config_dir, "config.json")) as f:
+        config = json.load(f)
+
+    return config["full_build"]
+
 def load_packages():
     packages = {}
 
diff --git a/devbot/git.py b/devbot/git.py
index 1e361f0..4a86bd6 100644
--- a/devbot/git.py
+++ b/devbot/git.py
@@ -2,6 +2,9 @@ import os
 import subprocess
 
 from devbot import command
+from devbot import utils
+
+_root_path = None
 
 def _chdir(func):
     def wrapped(*args, **kwargs):
@@ -72,6 +75,21 @@ class Module:
         return subprocess.check_output(["git", "describe"]).strip()
 
     @_chdir
+    def diff(self):
+        return subprocess.check_output(["git", "diff"])
+
+    @_chdir
+    def is_valid(self):
+        result = subprocess.call(["git", "rev-parse", "HEAD"],
+                                 stdout=utils.devnull,
+                                 stderr=utils.devnull)
+        return result == 0
+
+    @_chdir
+    def get_commit_id(self):
+        return subprocess.check_output(["git", "rev-parse", "HEAD"]).strip()
+
+    @_chdir
     def get_annotation(self, tag):
         # FIXME this is fragile, there must be a better way
 
@@ -101,3 +119,18 @@ class Module:
         command.run(["git", "clean", "-fdx"])
 
         return True
+
+def set_root_path(path):
+    global _root_path
+    _root_path = path
+
+def get_root_module():
+    remote = "git://git.sugarlabs.org/sugar-build/sugar-build.git"
+
+    module = Module(name=os.path.basename(_root_path),
+                    remote=remote,
+                    path=os.path.dirname(_root_path))
+    if not module.is_valid():
+        return None
+
+    return module
diff --git a/devbot/state.py b/devbot/state.py
index 258c594..b44652f 100644
--- a/devbot/state.py
+++ b/devbot/state.py
@@ -1,50 +1,93 @@
+import hashlib
 import os
 import json
 
 from devbot import config
+from devbot import git
 
-_state = None
+_BUILT_MODULES = "builtmodules"
+_FULL_BUILD = "fullbuild"
+_SYSTEM_CHECK = "syscheck"
 
-def _get_state_path():
-    return os.path.join(config.build_state_dir, "state.json")
+def _get_state_path(name):
+    return os.path.join(config.build_state_dir, "%s.json" % name)
 
-def _get_state():
-    global _state
+def _load_state(name, default=None):
+    state = default
 
-    if _state is not None:
-        return _state
+    try:
+        with open(_get_state_path(name)) as f:
+            state = json.load(f)
+    except IOError:
+        pass
+
+    return state
+
+def _save_state(name, state):
+    with open(_get_state_path(name), "w+") as f:
+        json.dump(state, f, indent=4)
+        f.write('\n')
+
+def _get_diff_hash(git_module):
+    diff = git_module.diff().strip()
+    if diff:
+        return hashlib.sha256(diff).hexdigest()
+    else:
+        return None
 
-    state_path = _get_state_path()
-    if os.path.exists(state_path):
-        _state = json.load(open(state_path))
+def _get_root_commit_id():
+    git_module = git.get_root_module()
+    if git_module:
+        commit_id = git_module.get_commit_id()
     else:
-        _state = { "built_modules": {} }
+        commit_id = "snapshot"
+
+    return commit_id
+
+def built_module_touch(module):
+    git_module = module.get_git_module()
+    built_modules = _load_state(_BUILT_MODULES, {})
+
+    info = {"commit": git_module.get_commit_id(),
+            "diff_hash": _get_diff_hash(git_module)}
+    built_modules[module.name] = info
+
+    _save_state(_BUILT_MODULES, built_modules)
+
+def built_module_is_unchanged(module):
+    git_module = module.get_git_module()
+    built_modules = _load_state(_BUILT_MODULES, {})
+    if module.name not in built_modules:
+        return False
 
-    return _state
+    info = built_modules[module.name]
 
-def _state_changed():
-    json.dump(_state, open(_get_state_path(), "w+"))
+    return info["diff_hash"] == _get_diff_hash(git_module) and \
+           info["commit"] == git_module.get_commit_id()
 
-def touch_built_commit_id(module):
-    _get_state()["built_modules"][module.name] = module.get_commit_id()
-    _state_changed()
+def system_check_is_unchanged():
+    system_check = _load_state(_SYSTEM_CHECK)
+    if not system_check:
+        return False
 
-def remove_built_commit_id(module):
-    state = _get_state()
+    return system_check["commit"] == _get_root_commit_id()
 
-    if module.name in state["built_modules"]:
-        del state["built_modules"][module.name]
-        _state_changed()
+def system_check_touch():
+    system_check = _load_state(_SYSTEM_CHECK, {})
+    system_check["commit"] = _get_root_commit_id()
+    _save_state(_SYSTEM_CHECK, system_check)
 
-def get_built_commit_id(module):
-    return _get_state()["built_modules"].get(module.name, None)
+def full_build_is_required():
+    full_build = _load_state(_FULL_BUILD)
+    if not full_build:
+        return True
 
-def get_last_system_check():
-    return _get_state().get("last_system_check", None)
+    return not (full_build["last"] == config.get_full_build())
 
-def touch_last_system_check():
-    _get_state()["last_system_check"] = config.get_commit_id()
-    _state_changed()
+def full_build_touch():
+    full_build = _load_state(_FULL_BUILD, {})
+    full_build["last"] = config.get_full_build()
+    _save_state(_FULL_BUILD, full_build)
 
 def clean():
     _state = None
@@ -52,6 +95,7 @@ def clean():
     print "Deleting state"
 
     try:
-        os.unlink(_get_state_path())
+        for name in _BUILT_MODULES, _SYSTEM_CHECK, _FULL_BUILD:
+            os.unlink(_get_state_path(name))
     except OSError:
         pass
diff --git a/devbot/system.py b/devbot/system.py
index 85e2349..4b014a8 100644
--- a/devbot/system.py
+++ b/devbot/system.py
@@ -4,6 +4,7 @@ import subprocess
 import sys
 
 from devbot import config
+from devbot import git
 from devbot import distro
 from devbot import command
 from devbot import state
@@ -156,8 +157,8 @@ def remove_packages(package_manager, packages):
 def check(remove=False, update=False, test=False, interactive=True,
           skip_if_unchanged=False):
     if skip_if_unchanged:
-        if config.get_commit_id() == state.get_last_system_check():
-            return
+        if state.system_check_is_unchanged():
+            return True
 
     package_manager = \
         distro.get_package_manager(test=test, interactive=interactive)
@@ -168,12 +169,12 @@ def check(remove=False, update=False, test=False, interactive=True,
 
     checks = config.load_prerequisites()
     if not run_checks(package_manager, checks, packages):
-        sys.exit(1)
+        return False
 
     xvfb_proc, orig_display = xvfb.start()
 
     if not run_checks(package_manager, config.load_checks(), packages):
-        sys.exit(1)
+        return False
 
     xvfb.stop(xvfb_proc, orig_display)
 
@@ -185,4 +186,6 @@ def check(remove=False, update=False, test=False, interactive=True,
     if remove:
         remove_packages(package_manager, packages)
 
-    state.touch_last_system_check()
+    state.system_check_touch()
+
+    return True
diff --git a/devbot/utils.py b/devbot/utils.py
index 2864948..aed5de0 100644
--- a/devbot/utils.py
+++ b/devbot/utils.py
@@ -1,18 +1 @@
-import os
-import subprocess
-
 devnull = open("/dev/null", "w")
-
-def get_commit_id(dir):
-    orig_cwd = os.getcwd()
-    os.chdir(dir)
-
-    try:
-        commit_id = subprocess.check_output(["git", "rev-parse", "HEAD"],
-                                            stderr=devnull).strip()
-    except subprocess.CalledProcessError:
-        commit_id = None
-
-    os.chdir(orig_cwd)
-
-    return commit_id