diff options
-rw-r--r-- | Makefile | 7 | ||||
-rwxr-xr-x | commands/build | 8 | ||||
-rwxr-xr-x | commands/clean | 8 | ||||
-rw-r--r-- | commands/common.py | 5 | ||||
-rwxr-xr-x | commands/run | 9 | ||||
-rwxr-xr-x | commands/shell | 8 | ||||
-rw-r--r-- | devbot/build.py | 167 | ||||
-rw-r--r-- | devbot/config.py | 66 | ||||
-rw-r--r-- | devbot/environ.py | 85 | ||||
-rw-r--r-- | devbot/run.py | 9 | ||||
-rw-r--r-- | devbot/shell.py | 11 | ||||
-rwxr-xr-x | scripts/dn-build | 282 | ||||
-rw-r--r-- | scripts/xinitrc | 3 |
13 files changed, 375 insertions, 293 deletions
@@ -2,7 +2,6 @@ TIMESTAMP := $(shell date +%Y%m%d-%H%M%S) LOGFILE = $(CURDIR)/logs/build-$(TIMESTAMP).log SCRIPTS = $(CURDIR)/scripts COMMANDS = $(CURDIR)/commands -DNBUILD = $(SCRIPTS)/dn-build LOG = $(SCRIPTS)/log-command # The buildbot shell does not handle script properly. It's unnecessary @@ -30,7 +29,7 @@ check-system: $(TYPESCRIPT) $(COMMANDS)/check-system $(LOGFILE) build: check-system - $(LOG) "$(DNBUILD) build" $(LOGFILE) + $(LOG) "$(COMMANDS)/build" $(LOGFILE) run: x11-utils $(SCRIPTS)/shell/start-sugar @@ -42,13 +41,13 @@ shell: x11-utils @PS1="[sugar-build \W]$$ " \ PATH=$(PATH):$(SCRIPTS)/shell \ SUGAR_BUILD_SHELL=yes \ - $(DNBUILD) shell + $(COMMANDS)/shell bug-report: @$(SCRIPTS)/bug-report clean: - $(DNBUILD) clean + $(COMMANDS)/clean rm -f logs/*.log logs/all-logs.tar.bz2 rm -f scripts/list-outputs rm -f scripts/find-free-display diff --git a/commands/build b/commands/build new file mode 100755 index 0000000..99f3048 --- /dev/null +++ b/commands/build @@ -0,0 +1,8 @@ +#!/usr/bin/python -u + +import common + +from devbot import build + +common.setup() +build.build() diff --git a/commands/clean b/commands/clean new file mode 100755 index 0000000..fc29466 --- /dev/null +++ b/commands/clean @@ -0,0 +1,8 @@ +#!/usr/bin/python + +import common + +from devbot import build + +common.setup() +build.clean() diff --git a/commands/common.py b/commands/common.py index 2836291..6f676a6 100644 --- a/commands/common.py +++ b/commands/common.py @@ -9,4 +9,7 @@ from devbot import system from devbot import config def setup(): - config.set_path(os.path.join(base_path, "scripts", "deps")) + config.set_config_dir(os.path.join(base_path, "scripts")) + config.set_install_dir(os.path.join(base_path, "install")) + config.set_source_dir(os.path.join(base_path, "source")) + config.set_build_dir(os.path.join(base_path, "build")) diff --git a/commands/run b/commands/run new file mode 100755 index 0000000..b1f76aa --- /dev/null +++ b/commands/run @@ -0,0 +1,9 @@ +#!/usr/bin/python + +import common +import sys + +from devbot import run + +common.setup() +run.run(sys.argv[1:]) diff --git a/commands/shell b/commands/shell new file mode 100755 index 0000000..e588e06 --- /dev/null +++ b/commands/shell @@ -0,0 +1,8 @@ +#!/usr/bin/python + +import common + +from devbot import shell + +common.setup() +shell.start() diff --git a/devbot/build.py b/devbot/build.py new file mode 100644 index 0000000..22c1ccf --- /dev/null +++ b/devbot/build.py @@ -0,0 +1,167 @@ +#!/usr/bin/python -u + +from distutils import sysconfig +import glob +import json +import os +import multiprocessing +import shutil +import sys +import subprocess + +from devbot import config +from devbot import environ + +state = { "built_modules": {} } + +def get_state_path(): + return os.path.join(config.devbot_dir, "state.json") + +def load_state(): + global state + + state_path = get_state_path() + if os.path.exists(state_path): + state = json.load(open(state_path)) + +def save_state(): + json.dump(state, open(get_state_path(), "w+")) + +def add_path(name, path): + if name not in os.environ: + os.environ[name] = path + return + + splitted = os.environ[name].split(":") + splitted.append(path) + + os.environ[name] = ":".join(splitted) + +def get_module_source_dir(module): + return os.path.join(config.source_dir, module["name"]) + +def get_module_build_dir(module): + return os.path.join(config.build_dir, module["name"]) + +def get_module_commit_id(module): + orig_cwd = os.getcwd() + os.chdir(get_module_source_dir(module)) + + commit_id = subprocess.check_output(["git", "rev-parse", "HEAD"]) + + os.chdir(orig_cwd) + + return commit_id.strip() + +def run_command(args): + print " ".join(args) + subprocess.check_call(args) + +def unlink_libtool_files(): + orig_cwd = os.getcwd() + os.chdir(config.lib_dir) + + for filename in glob.glob("*.la"): + os.unlink(filename) + + os.chdir(orig_cwd) + +def pull_source(module): + module_dir = get_module_source_dir(module) + + if os.path.exists(module_dir): + os.chdir(module_dir) + + run_command(["git", "remote", "set-url", "origin", module["repo"]]) + run_command(["git", "remote", "update", "origin"]) + else: + os.chdir(config.source_dir) + run_command(["git", "clone", "--progress", + module["repo"], module["name"]]) + os.chdir(module_dir) + + branch = module.get("branch", "master") + run_command(["git", "checkout", branch]) + +def build_autotools(module): + autogen = os.path.join(get_module_source_dir(module), "autogen.sh") + + jobs = multiprocessing.cpu_count() * 2 + + run_command([autogen, + "--prefix", config.install_dir, + "--libdir", config.lib_dir]) + + run_command(["make", "-j", "%d" % jobs]) + run_command(["make", "install"]) + + unlink_libtool_files() + +def build_activity(module): + run_command(["./setup.py", "install", "--prefix", config.install_dir]) + +def build_module(module): + module_source_dir = get_module_source_dir(module) + + if module.get("out-of-source", True): + module_build_dir = get_module_build_dir(module) + + if not os.path.exists(module_build_dir): + os.mkdir(module_build_dir) + + os.chdir(module_build_dir) + else: + os.chdir(module_source_dir) + + if os.path.exists(os.path.join(module_source_dir, "setup.py")): + build_activity(module) + elif os.path.exists(os.path.join(module_source_dir, "autogen.sh")): + build_autotools(module) + else: + print "Unknown build system" + sys.exit(1) + + state["built_modules"][module["name"]] = get_module_commit_id(module) + save_state() + +def clear_built_modules(modules, index): + if index < len(modules) - 1: + for module in modules[index + 1:]: + name = module["name"] + if name in state["built_modules"]: + del state["built_modules"][name] + +def rmtree(dir): + print "Deleting %s" % dir + shutil.rmtree(dir, ignore_errors=True) + +def build(): + environ.setup() + load_state() + + modules = config.load_modules() + + for i, module in enumerate(modules): + print "\n=== Building %s ===\n" % module["name"] + + try: + pull_source(module) + + old_commit_id = state["built_modules"].get(module["name"], None) + new_commit_id = get_module_commit_id(module) + + if old_commit_id is None or old_commit_id != new_commit_id: + clear_built_modules(modules, i) + build_module(module) + else: + print "\n* Already built, skipping *" + except subprocess.CalledProcessError: + sys.exit(1) + +def clean(): + rmtree(config.install_dir) + rmtree(config.build_dir) + + for module in config.load_modules(): + if not module.get("out-of-source", True): + rmtree(get_module_source_dir(module)) diff --git a/devbot/config.py b/devbot/config.py index b63c6e6..11a861c 100644 --- a/devbot/config.py +++ b/devbot/config.py @@ -3,11 +3,52 @@ import os from devbot import distro -config_path = None +config_dir = None +install_dir = None +source_dir = None +build_dir = None +lib_dir = None +devbot_dir = None +share_dir = None +bin_dir = None +etc_dir = None +use_lib64 = os.uname()[4] == "x86_64" -def set_path(path): - global config_path - config_path = path +if use_lib64: + system_lib_dir = "/usr/lib64" +else: + system_lib_dir = "/usr/lib" + +def set_config_dir(dir): + global config_dir + config_dir = dir + +def set_install_dir(dir): + global install_dir + global devbot_dir + global share_dir + global bin_dir + global etc_dir + global lib_dir + + install_dir = dir + devbot_dir = os.path.join(install_dir, "devbot") + share_dir = os.path.join(install_dir, "share") + bin_dir = os.path.join(install_dir, "bin") + etc_dir = os.path.join(install_dir, "etc") + + if use_lib64: + lib_dir = os.path.join(install_dir, "lib64") + else: + lib_dir = os.path.join(install_dir, "lib") + +def set_source_dir(dir): + global source_dir + source_dir = dir + +def set_build_dir(dir): + global build_dir + build_dir = dir def load_packages(): packages = _load_deps_json("packages-%s" % distro.get_system_version()) @@ -26,7 +67,22 @@ def load_checks(): return checks +def load_modules(): + version = distro.get_system_version() + + module_files = ["system-%s.json" % version, + "sugar.json", + "activities.json"] + + modules = [] + + for module_file in module_files: + path = os.path.join(config_dir, "modules", module_file) + modules.extend(json.load(open(path))) + + return modules + def _load_deps_json(name): - path = os.path.join(config_path, "%s.json" % name) + path = os.path.join(config_dir, "deps", "%s.json" % name) return json.load(open(path)) diff --git a/devbot/environ.py b/devbot/environ.py new file mode 100644 index 0000000..d18370d --- /dev/null +++ b/devbot/environ.py @@ -0,0 +1,85 @@ +from distutils import sysconfig +import os + +from devbot import config + +def setup(): + _setup_dirs() + _setup_gconf() + _setup_variables() + +def _add_path(name, path): + if name not in os.environ: + os.environ[name] = path + return + + splitted = os.environ[name].split(":") + splitted.append(path) + + os.environ[name] = ":".join(splitted) + +def _setup_variables(): + _add_path("LD_LIBRARY_PATH", config.lib_dir) + _add_path("PATH", config.bin_dir) + + _add_path("GIO_EXTRA_MODULES", + os.path.join(config.system_lib_dir, "gio", "modules")) + _add_path("GI_TYPELIB_PATH", + os.path.join(config.lib_dir, "girepository-1.0")) + _add_path("GI_TYPELIB_PATH", + os.path.join(config.system_lib_dir, "girepository-1.0")) + _add_path("PKG_CONFIG_PATH", + os.path.join(config.lib_dir, "pkgconfig")) + _add_path("GST_PLUGIN_PATH", + os.path.join(config.lib_dir , "gstreamer-1.0")) + _add_path("GST_REGISTRY", + os.path.join(config.devbot_dir, "gstreamer.registry")) + _add_path("PYTHONPATH", + sysconfig.get_python_lib(prefix=config.install_dir)) + _add_path("PYTHONPATH", + sysconfig.get_python_lib(prefix=config.install_dir, + plat_specific=True)) + + _add_path("XDG_DATA_DIRS", "/usr/share") + _add_path("XDG_DATA_DIRS", config.share_dir) + + _add_path("XDG_CONFIG_DIRS", "/etc") + _add_path("XDG_CONFIG_DIRS", config.etc_dir) + + os.environ["GTK_DATA_PREFIX"] = config.install_dir + os.environ["GTK_PATH"] = os.path.join(config.lib_dir, "gtk-2.0") + +def _setup_gconf(): + gconf_dir = os.path.join(config.etc_dir, "gconf") + gconf_pathdir = os.path.join(gconf_dir, "2") + + if not os.path.exists(gconf_pathdir): + os.makedirs(gconf_pathdir) + + gconf_path = os.path.join(gconf_pathdir, "path.jhbuild") + if not os.path.exists(gconf_path): + input = open("/etc/gconf/2/path") + output = open(gconf_path, "w") + + for line in input.readlines(): + if "/etc/gconf" in line: + output.write(line.replace("/etc/gconf", gconf_dir)) + output.write(line) + + output.close() + input.close() + + os.environ["GCONF_DEFAULT_SOURCE_PATH"] = gconf_path + + os.environ["GCONF_SCHEMA_INSTALL_SOURCE"] = \ + "xml:merged:" + os.path.join(gconf_dir, "gconf.xml.defaults") + +def _setup_dirs(): + for dir in [config.source_dir, + config.install_dir, + config.build_dir, + config.share_dir, + config.devbot_dir, + os.path.join(config.share_dir, "aclocal")]: + if not os.path.exists(dir): + os.mkdir(dir) diff --git a/devbot/run.py b/devbot/run.py new file mode 100644 index 0000000..c27862f --- /dev/null +++ b/devbot/run.py @@ -0,0 +1,9 @@ +#!/usr/bin/python -u + +import os + +from devbot import environ + +def run(args): + environ.setup() + os.execlp(args[0], *args) diff --git a/devbot/shell.py b/devbot/shell.py new file mode 100644 index 0000000..abe0695 --- /dev/null +++ b/devbot/shell.py @@ -0,0 +1,11 @@ +#!/usr/bin/python + +import os + +from devbot import environ + +def start(): + environ.setup() + + user_shell = os.environ.get("SHELL", "/bin/sh") + os.execlp(user_shell, user_shell) diff --git a/scripts/dn-build b/scripts/dn-build deleted file mode 100755 index fb4bae8..0000000 --- a/scripts/dn-build +++ /dev/null @@ -1,282 +0,0 @@ -#!/usr/bin/python -u - -from distutils import sysconfig -import glob -import json -import os -import multiprocessing -import shutil -import sys -import subprocess - -import sysinfo - -system_version = sysinfo.get_system_version() -module_files = ["system-%s.json" % system_version, - "sugar.json", - "activities.json"] - -scripts_dir = os.path.abspath(os.path.dirname(__file__)) -base_dir = os.path.dirname(scripts_dir) -install_dir = os.path.join(base_dir, "install") -dnbuild_dir = os.path.join(install_dir, "dnbuild") -share_dir = os.path.join(install_dir, "share") -include_dir = os.path.join(install_dir, "include") -bin_dir = os.path.join(install_dir, "bin") -etc_dir = os.path.join(install_dir, "etc") -build_dir = os.path.join(base_dir, "build") -source_dir = os.path.join(base_dir, "source") -modules_dir = os.path.join(scripts_dir, "modules") -build_state_path = os.path.join(dnbuild_dir, "state.json") - -if os.uname()[4] == "x86_64": - lib_dir = os.path.join(install_dir, "lib64") - system_lib_dir = "/usr/lib64" -else: - lib_dir = os.path.join(install_dir, "lib") - system_lib_dir = "/usr/lib" - -state = { "built_modules": {} } - -def load_state(): - global state - - if os.path.exists(build_state_path): - state = json.load(open(build_state_path)) - -def save_state(): - json.dump(state, open(build_state_path, "w+")) - -def add_path(name, path): - if name not in os.environ: - os.environ[name] = path - return - - splitted = os.environ[name].split(":") - splitted.append(path) - - os.environ[name] = ":".join(splitted) - -def get_module_source_dir(module): - return os.path.join(source_dir, module["name"]) - -def get_module_build_dir(module): - return os.path.join(build_dir, module["name"]) - -def get_module_commit_id(module): - orig_cwd = os.getcwd() - os.chdir(get_module_source_dir(module)) - - commit_id = subprocess.check_output(["git", "rev-parse", "HEAD"]) - - os.chdir(orig_cwd) - - return commit_id.strip() - -def run_command(args): - print " ".join(args) - subprocess.check_call(args) - -def unlink_libtool_files(): - orig_cwd = os.getcwd() - os.chdir(lib_dir) - - for filename in glob.glob("*.la"): - os.unlink(filename) - - os.chdir(orig_cwd) - -def pull_source(module): - module_dir = get_module_source_dir(module) - - if os.path.exists(module_dir): - os.chdir(module_dir) - - run_command(["git", "remote", "set-url", "origin", module["repo"]]) - run_command(["git", "remote", "update", "origin"]) - else: - os.chdir(source_dir) - run_command(["git", "clone", "--progress", - module["repo"], module["name"]]) - os.chdir(module_dir) - - branch = module.get("branch", "master") - run_command(["git", "checkout", branch]) - -def build_autotools(module): - autogen = os.path.join(get_module_source_dir(module), "autogen.sh") - - jobs = multiprocessing.cpu_count() * 2 - - run_command([autogen, - "--prefix", install_dir, - "--libdir", lib_dir]) - - run_command(["make", "-j", "%d" % jobs]) - run_command(["make", "install"]) - - unlink_libtool_files() - -def build_activity(module): - run_command(["./setup.py", "install", "--prefix", install_dir]) - -def build(module): - module_source_dir = get_module_source_dir(module) - - if module.get("out-of-source", True): - module_build_dir = get_module_build_dir(module) - - if not os.path.exists(module_build_dir): - os.mkdir(module_build_dir) - - os.chdir(module_build_dir) - else: - os.chdir(module_source_dir) - - if os.path.exists(os.path.join(module_source_dir, "setup.py")): - build_activity(module) - elif os.path.exists(os.path.join(module_source_dir, "autogen.sh")): - build_autotools(module) - else: - print "Unknown build system" - sys.exit(1) - - state["built_modules"][module["name"]] = get_module_commit_id(module) - save_state() - -def load_modules(): - modules = [] - - for module_file in module_files: - path = os.path.join(modules_dir, module_file) - modules.extend(json.load(open(path))) - - return modules - -def clear_built_modules(modules, index): - if index < len(modules) - 1: - for module in modules[index + 1:]: - name = module["name"] - if name in state["built_modules"]: - del state["built_modules"][name] - -def rmtree(dir): - print "Deleting %s" % dir - shutil.rmtree(dir, ignore_errors=True) - -def cmd_build(): - modules = load_modules() - - for i, module in enumerate(modules): - print "\n=== Building %s ===\n" % module["name"] - - try: - pull_source(module) - - old_commit_id = state["built_modules"].get(module["name"], None) - new_commit_id = get_module_commit_id(module) - - if old_commit_id is None or old_commit_id != new_commit_id: - clear_built_modules(modules, i) - build(module) - else: - print "\n* Already built, skipping *" - except subprocess.CalledProcessError: - sys.exit(1) - -def cmd_shell(): - user_shell = os.environ.get('SHELL', '/bin/sh') - os.execlp(user_shell, user_shell) - -def cmd_run(): - os.execlp(sys.argv[2], *sys.argv[2:]) - -def cmd_clean(): - rmtree(install_dir) - rmtree(build_dir) - - for module in load_modules(): - if not module.get("out-of-source", True): - rmtree(get_module_source_dir(module)) - -def setup_environ(): - add_path("LD_LIBRARY_PATH", lib_dir) - add_path("PATH", bin_dir) - - add_path("GIO_EXTRA_MODULES", - os.path.join(system_lib_dir, "gio", "modules")) - add_path("GI_TYPELIB_PATH", - os.path.join(lib_dir, "girepository-1.0")) - add_path("GI_TYPELIB_PATH", - os.path.join(system_lib_dir, "girepository-1.0")) - add_path("PKG_CONFIG_PATH", - os.path.join(lib_dir, "pkgconfig")) - add_path("GST_PLUGIN_PATH", - os.path.join(lib_dir , "gstreamer-1.0")) - add_path("GST_REGISTRY", - os.path.join(dnbuild_dir, "gstreamer.registry")) - add_path("PYTHONPATH", - sysconfig.get_python_lib(prefix=install_dir)) - add_path("PYTHONPATH", - sysconfig.get_python_lib(prefix=install_dir, plat_specific=True)) - - add_path("XDG_DATA_DIRS", "/usr/share") - add_path("XDG_DATA_DIRS", share_dir) - - add_path("XDG_CONFIG_DIRS", "/etc") - add_path("XDG_CONFIG_DIRS", etc_dir) - - os.environ["GTK_DATA_PREFIX"] = install_dir - os.environ["GTK_PATH"] = os.path.join(lib_dir, "gtk-2.0") - -def setup_gconf(): - gconf_dir = os.path.join(etc_dir, "gconf") - gconf_pathdir = os.path.join(gconf_dir, "2") - - if not os.path.exists(gconf_pathdir): - os.makedirs(gconf_pathdir) - - gconf_path = os.path.join(gconf_pathdir, "path.jhbuild") - if not os.path.exists(gconf_path): - input = open("/etc/gconf/2/path") - output = open(gconf_path, "w") - - for line in input.readlines(): - if "/etc/gconf" in line: - output.write(line.replace("/etc/gconf", gconf_dir)) - output.write(line) - - output.close() - input.close() - - os.environ["GCONF_DEFAULT_SOURCE_PATH"] = gconf_path - - os.environ["GCONF_SCHEMA_INSTALL_SOURCE"] = \ - "xml:merged:" + os.path.join(gconf_dir, "gconf.xml.defaults") - -def setup_dirs(): - for dir in [source_dir, - install_dir, - build_dir, - share_dir, - dnbuild_dir, - os.path.join(share_dir, "aclocal")]: - if not os.path.exists(dir): - os.mkdir(dir) - -def main(): - load_state() - - setup_dirs() - setup_gconf() - setup_environ() - - commands = {"build": cmd_build, - "shell": cmd_shell, - "run": cmd_run, - "clean": cmd_clean} - - if sys.argv[1]: - commands[sys.argv[1]]() - -main() diff --git a/scripts/xinitrc b/scripts/xinitrc index 686bbe6..c0339aa 100644 --- a/scripts/xinitrc +++ b/scripts/xinitrc @@ -6,6 +6,7 @@ unset SESSION_MANAGER SCRIPTS_DIR=`dirname "$0"` ROOT_DIR=`dirname "$SCRIPTS_DIR"` +COMMANDS_DIR=$ROOT_DIR/commands if [ ! -z $SUGAR_PROFILE ]; then grep -q PROFILE $ROOT_DIR/config @@ -55,7 +56,7 @@ fi RUN_COMMAND="dbus-launch --exit-with-session $SCRIPTS_DIR/run-with-keyring sugar" if [ -z $SUGAR_BUILD_SHELL ]; then - $SCRIPTS_DIR/dn-build run $RUN_COMMAND + $COMMANDS_DIR/run $RUN_COMMAND else $RUN_COMMAND fi |