diff options
author | Aleksey Lim <alsroot@sugarlabs.org> | 2012-09-22 06:32:59 (GMT) |
---|---|---|
committer | Aleksey Lim <alsroot@sugarlabs.org> | 2012-09-22 06:32:59 (GMT) |
commit | 046be81cb435826abbecff2cfb08b217ee742179 (patch) | |
tree | 473710d78f14bda66dd8d8c4d32545d1b7eb28fc | |
parent | 5d64afb1535b804d2dacd3513698422e31bc2c4e (diff) |
While launching, install packaged dependncies via PackageKit
-rw-r--r-- | TODO | 5 | ||||
-rw-r--r-- | sugar_network/__init__.py | 12 | ||||
-rw-r--r-- | sugar_network/local/mounts.py | 13 | ||||
-rw-r--r-- | sugar_network/node/commands.py | 7 | ||||
-rw-r--r-- | sugar_network/toolkit/http.py | 14 | ||||
-rw-r--r-- | sugar_network/zerosugar/feeds.py | 47 | ||||
-rw-r--r-- | sugar_network/zerosugar/injector.py | 209 | ||||
-rw-r--r-- | sugar_network/zerosugar/lsb_release.py | 164 | ||||
-rw-r--r-- | sugar_network/zerosugar/packagekit.py | 262 | ||||
-rw-r--r-- | sugar_network/zerosugar/pipe.py | 185 | ||||
-rw-r--r-- | sugar_network/zerosugar/solution.py | 63 | ||||
-rwxr-xr-x | tests/units/home_mount.py | 65 | ||||
-rwxr-xr-x | tests/units/injector.py | 205 | ||||
-rwxr-xr-x | tests/units/node.py | 26 | ||||
-rwxr-xr-x | tests/units/node_mount.py | 20 |
15 files changed, 999 insertions, 298 deletions
@@ -15,6 +15,11 @@ - process client configuration in more general manner than client stats sharing - i18n activity.info's strings - return 304 Not Modified from Router when it makes sense +- cache Solutions +- more consistent launch; right now + - zerosugar/pipe.py uses several mountpoints + - what mountpoints for deps +- if feed contains regular implementations and packages, solve() will reuse implementations and there is no way to use packages 1.0 === diff --git a/sugar_network/__init__.py b/sugar_network/__init__.py index 7ace3f2..350e332 100644 --- a/sugar_network/__init__.py +++ b/sugar_network/__init__.py @@ -17,7 +17,16 @@ from sugar_network.toolkit import sugar from sugar_network.local.activities import checkins from sugar_network.local import api_url, server_mode from sugar_network_webui import webui_port -from sugar_network.zerosugar.injector import launch, checkin + + +def launch(*args, **kwargs): + from sugar_network.zerosugar import injector + return injector.launch(*args, **kwargs) + + +def checkin(*args, **kwargs): + from sugar_network.zerosugar import injector + return injector.checkin(*args, **kwargs) def Client(url=None, **kwargs): @@ -34,6 +43,5 @@ def IPCClient(**kwargs): def DBusClient(*args, **kwargs): - # Avoid importing dbus related modules by default from sugar_network.local import dbus_client return dbus_client.DBusClient(*args, **kwargs) diff --git a/sugar_network/local/mounts.py b/sugar_network/local/mounts.py index c3c2745..d826afe 100644 --- a/sugar_network/local/mounts.py +++ b/sugar_network/local/mounts.py @@ -14,7 +14,6 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import os -import json import shutil import logging from os.path import isabs, exists, join, basename, isdir @@ -118,7 +117,7 @@ class HomeMount(LocalMount): @ad.property_command(method='GET', cmd='get_blob') def get_blob(self, document, guid, prop, request=None): if document == 'context' and prop == 'feed': - return json.dumps(self._get_feed(request)) + return self._get_feed(request) elif document == 'implementation' and prop == 'data': path = activities.guid_to_path(guid) if exists(path): @@ -127,7 +126,7 @@ class HomeMount(LocalMount): return LocalMount.get_blob(self, document, guid, prop, request) def _get_feed(self, request): - feed = {} + versions = {} for path in activities.checkins(request['guid']): try: @@ -141,7 +140,7 @@ class HomeMount(LocalMount): else: impl_id = activities.path_to_guid(spec.root) - feed[spec['version']] = { + versions[spec['version']] = { '*-*': { 'guid': impl_id, 'stability': 'stable', @@ -150,10 +149,14 @@ class HomeMount(LocalMount): 'exec': spec['Activity', 'exec'], }, }, + 'requires': spec.requires, }, } - return feed + if not versions: + return None + else: + return {'versions': versions} def _events_cb(self, event): found_commons = False diff --git a/sugar_network/node/commands.py b/sugar_network/node/commands.py index 5a6fc13..bcd8fe2 100644 --- a/sugar_network/node/commands.py +++ b/sugar_network/node/commands.py @@ -102,6 +102,13 @@ class NodeCommands(ad.VolumeCommands, Commands): else: props['user'] = [request.principal] self._set_author(props) + + implement = props.get('implement') + if self._is_master and implement: + if not isinstance(implement, basestring): + implement = implement[0] + props['guid'] = implement + ad.VolumeCommands.before_create(self, request, props) def before_update(self, request, props): diff --git a/sugar_network/toolkit/http.py b/sugar_network/toolkit/http.py index 36b1bfb..860a7d9 100644 --- a/sugar_network/toolkit/http.py +++ b/sugar_network/toolkit/http.py @@ -47,8 +47,8 @@ _logger = logging.getLogger('http') class Client(object): def __init__(self, api_url, **kwargs): - self._api_url = api_url - self._params = kwargs + self.api_url = api_url + self.params = kwargs verify = True if local.no_check_certificate.value: @@ -68,21 +68,21 @@ class Client(object): self._session = Session(headers=headers, verify=verify, prefetch=False) def get(self, path_=None, **kwargs): - kwargs.update(self._params) + kwargs.update(self.params) return self.request('GET', path_, params=kwargs) def post(self, path_=None, data_=None, **kwargs): - kwargs.update(self._params) + kwargs.update(self.params) return self.request('POST', path_, data_, headers={'Content-Type': 'application/json'}, params=kwargs) def put(self, path_=None, data_=None, **kwargs): - kwargs.update(self._params) + kwargs.update(self.params) return self.request('PUT', path_, data_, headers={'Content-Type': 'application/json'}, params=kwargs) def delete(self, path_=None, **kwargs): - kwargs.update(self._params) + kwargs.update(self.params) return self.request('DELETE', path_, params=kwargs) def request(self, method, path=None, data=None, headers=None, **kwargs): @@ -186,7 +186,7 @@ class Client(object): if not path: path = [''] if not isinstance(path, basestring): - path = '/'.join([i.strip('/') for i in [self._api_url] + path]) + path = '/'.join([i.strip('/') for i in [self.api_url] + path]) if data is not None and headers and \ headers.get('Content-Type') == 'application/json': diff --git a/sugar_network/zerosugar/feeds.py b/sugar_network/zerosugar/feeds.py index 0fb5558..8cc60ac 100644 --- a/sugar_network/zerosugar/feeds.py +++ b/sugar_network/zerosugar/feeds.py @@ -20,6 +20,7 @@ from os.path import isabs from zeroinstall.injector import model import sweets_recipe +from sugar_network.zerosugar import lsb_release from active_toolkit import util, enforce @@ -36,20 +37,31 @@ def read(context): for client in clients: try: blob = client.get(['context', context, 'feed'], cmd='get_blob') - enforce(blob and 'path' in blob, 'No feed for %r context', context) - with file(blob['path']) as f: - feed_content = json.load(f) - if feed_content: - break + enforce(blob is not None, 'Feed not found') + if 'path' in blob: + with file(blob['path']) as f: + feed_content = json.load(f) + else: + feed_content = blob + _logger.debug('Found %r feed in %r mountpoint', + context, client.params['mountpoint']) + break except Exception: util.exception(_logger, - 'Failed to fetch feed for %r context', context) + 'Failed to fetch %r feed from %r mountpoint', + context, client.params['mountpoint']) if feed_content is None: _logger.warning('No feed for %r context', context) return None - for version, version_data in feed_content.items(): + packages = feed_content.get('packages') or {} + distro = packages.get(lsb_release.distributor_id()) + if distro: + feed.to_resolve = distro.get('binary') + + versions = feed_content.get('versions') or {} + for version, version_data in versions.items(): for arch, impl_data in version_data.items(): impl_id = impl_data['guid'] @@ -60,8 +72,7 @@ def read(context): impl.arch = arch impl.upstream_stability = \ model.stability_levels[impl_data['stability']] - # TODO - #impl.requires.extend(_read_requires(impl_data.get('requires'))) + impl.requires.extend(_read_requires(impl_data.get('requires'))) if isabs(impl_id): impl.local_path = impl_id @@ -92,6 +103,7 @@ class _Feed(model.ZeroInstallFeed): self.feeds = [] self.metadata = [] self.last_checked = None + self.to_resolve = None self._package_implementations = [] @property @@ -124,10 +136,27 @@ class _Feed(model.ZeroInstallFeed): def first_description(self): return self.context + def resolve(self, packages): + top_package = packages[0] + + impl = _Implementation(self, self.context, None) + impl.version = sweets_recipe.parse_version(top_package['version']) + impl.released = 0 + impl.arch = '*-%s' % top_package['arch'] + impl.upstream_stability = model.stability_levels['packaged'] + impl.to_install = [i for i in packages if not i['installed']] + + self.implementations[self.context] = impl + self.to_resolve = None + class _Implementation(model.ZeroInstallImplementation): client = None + to_install = None + + def is_available(self, stores=None): + return self.to_install is not None or bool(self.local_path) class _Dependency(model.InterfaceDependency): diff --git a/sugar_network/zerosugar/injector.py b/sugar_network/zerosugar/injector.py index 4a1080b..df30dc9 100644 --- a/sugar_network/zerosugar/injector.py +++ b/sugar_network/zerosugar/injector.py @@ -13,45 +13,31 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -# pylint: disable-msg=W0212 - import os -import sys -import json -import signal import shutil import logging -import threading from os.path import join, exists, basename from zeroinstall.injector import model -from zeroinstall.injector.config import Config -from zeroinstall.injector.driver import Driver from zeroinstall.injector.requirements import Requirements from sweets_recipe import Spec -from sugar_network.zerosugar import feeds -from sugar_network.zerosugar.solution import Solution +from sugar_network.zerosugar import pipe, packagekit +from sugar_network.zerosugar.solution import solve from sugar_network import local from sugar_network.toolkit import sugar -from active_toolkit import coroutine, util, enforce +from active_toolkit import util, enforce _logger = logging.getLogger('zerosugar.injector') -_pipe = None def launch(mountpoint, context, command='activity', args=None): - return _fork(_launch, mountpoint, context, command, args) + return pipe.fork(_launch, mountpoint, context, command, args) def checkin(mountpoint, context, command='activity'): - return _fork(_checkin, mountpoint, context, command) - - -def _progress(**event): - os.write(_pipe, json.dumps(event)) - os.write(_pipe, '\n') + return pipe.fork(_checkin, mountpoint, context, command) def _launch(mountpoint, context, command, args): @@ -63,7 +49,7 @@ def _launch(mountpoint, context, command, args): args = cmd.path.split() + args _logger.info('Executing %s: %s', solution.interface, args) - _progress(state='exec') + pipe.progress('exec') if command == 'activity': _activity_env(solution.top, os.environ) @@ -87,40 +73,21 @@ def _checkin(mountpoint, context, command): raise -def _solve(req): - driver = Driver(Config(), req) - - driver.solver.solve(req.interface_uri, - driver.target_arch, command_name=req.command) - - result = Solution(driver.solver.selections, req) - result.details = dict((k.uri, v) - for k, v in (driver.solver.details or {}).items()) - result.ready = driver.solver.ready - - if not result.ready: - # pylint: disable-msg=W0212 - failure_reason = driver.solver._failure_reason - if not failure_reason: - missed_ifaces = [iface.uri for iface, impl in - driver.solver.selections.items() if impl is None] - failure_reason = 'Cannot find requireed implementations ' \ - 'for %s' % ', '.join(missed_ifaces) - result.failure_reason = model.SafeException(failure_reason) - - return result - - def _make(context, command): requirement = Requirements(context) requirement.command = command - _progress(state='analyze') - solution = _solve(requirement) - enforce(solution.ready, solution.failure_reason) + pipe.progress('analyze') + solution = solve(requirement) + + to_install = [] + for sel, __, __ in solution.walk(): + to_install.extend(sel.to_install or []) + if to_install: + packagekit.install(to_install) for sel, __, __ in solution.walk(): - if sel.local_path: + if sel.is_available(): continue enforce(sel.download_sources, @@ -128,7 +95,7 @@ def _make(context, command): sel.interface) # TODO Per download progress - _progress(state='download') + pipe.progress('download') impl = sel.client.get(['implementation', sel.id, 'data'], cmd='get_blob') @@ -140,7 +107,7 @@ def _make(context, command): impl_path = join(impl_path, dl.extract) sel.local_path = impl_path - _progress(state='ready', session={'implementation': solution.top.id}) + pipe.progress('ready', session={'implementation': solution.top.id}) return solution @@ -168,145 +135,3 @@ def _activity_env(selection, environ): environ['SUGAR_LOCALEDIR'] = join(selection.local_path, 'locale') os.chdir(selection.local_path) - - -def _setup_logging(context): - log_dir = sugar.profile_path('logs') - if not exists(log_dir): - os.makedirs(log_dir) - path = util.unique_filename(log_dir, context + '.log') - - def stdfd(stream): - if hasattr(stream, 'fileno'): - return stream.fileno() - else: - # Sugar Shell wraps std streams - return stream._stream.fileno() - - logfile = file(path, 'a+') - os.dup2(logfile.fileno(), stdfd(sys.stdout)) - os.dup2(logfile.fileno(), stdfd(sys.stderr)) - logfile.close() - - debug = sugar.logger_level() - if not debug: - level = logging.WARNING - elif debug == 1: - level = logging.INFO - else: - level = logging.DEBUG - - root_logger = logging.getLogger() - for handler in root_logger.handlers: - root_logger.removeHandler(handler) - logging.basicConfig(level=level, - format='%(asctime)s %(levelname)s %(name)s: %(message)s') - - return path - - -def _decode_exit_failure(status): - failure = None - if os.WIFEXITED(status): - status = os.WEXITSTATUS(status) - if status: - failure = 'Exited with status %s' % status - elif os.WIFSIGNALED(status): - signum = os.WTERMSIG(status) - if signum not in (signal.SIGINT, signal.SIGKILL, signal.SIGTERM): - failure = 'Terminated by signal %s' % signum - else: - signum = os.WTERMSIG(status) - failure = 'Undefined status with signal %s' % signum - return failure - - -def _fork(callback, mountpoint, context, *args): - fd_r, fd_w = os.pipe() - - pid = os.fork() - if pid: - os.close(fd_w) - return _Pipe(pid, fd_r) - - from sugar_network import IPCClient - - os.close(fd_r) - global _pipe - _pipe = fd_w - - def thread_func(): - _progress(state='boot', - session={ - 'log_path': _setup_logging(context), - 'mountpoint': mountpoint, - 'context': context, - }) - - feeds.clients.append(IPCClient(mountpoint='~')) - if mountpoint != '~': - feeds.clients.append(IPCClient(mountpoint=mountpoint)) - - try: - callback(mountpoint, context, *args) - except Exception, error: - util.exception(_logger) - _progress(state='failure', error=str(error)) - - # Avoid a mess with current thread coroutines - thread = threading.Thread(target=thread_func) - thread.start() - thread.join() - - os.close(fd_w) - sys.stdout.flush() - sys.stderr.flush() - os._exit(0) - - -class _Pipe(object): - - def __init__(self, pid, fd): - self._pid = pid - self._file = os.fdopen(fd) - self._session = {} - - def fileno(self): - return None if self._file is None else self._file.fileno() - - def read(self): - if self._file is None: - return None - - event = self._file.readline() - if not event: - status = 0 - try: - __, status = os.waitpid(self._pid, 0) - except OSError: - pass - failure = _decode_exit_failure(status) - if failure: - event = {'state': 'failure', 'error': failure} - event.update(self._session) - return event - else: - self._file.close() - self._file = None - return None - - event = json.loads(event) - if 'session' in event: - self._session.update(event.pop('session')) - event.update(self._session) - return event - - def __iter__(self): - if self._file is None: - return - while True: - coroutine.select([self._file.fileno()], [], []) - event = self.read() - if event is None: - break - yield event diff --git a/sugar_network/zerosugar/lsb_release.py b/sugar_network/zerosugar/lsb_release.py new file mode 100644 index 0000000..47072d8 --- /dev/null +++ b/sugar_network/zerosugar/lsb_release.py @@ -0,0 +1,164 @@ +# Copyright (C) 2011 Aleksey Lim +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +"""Get LSB (Linux Standard Base) Distribution information. + +$Repo: git://git.sugarlabs.org/alsroot/codelets.git$ +$File: src/lsb_release.py$ +$Date: 2012-08-13$ + +""" + +import re +import math +import subprocess +from os.path import exists + + +_distributor_id = None +_release = None + +_DERIVATES = { + 'Trisquel': ( + 'Ubuntu', [ + lambda x: '%02d.%02d' % + (int(float(x)) + 6, + 4 if float(x) - int(float(x)) < 0.5 else 10), + ], + ), + 'LinuxMint': ( + 'Ubuntu', [ + lambda x: '%02d.%02d' % + (math.ceil(int(x) / 2.) + 5, + [4, 10][(int(x) - 1) % 2]), + ], + ), + } + + +def distributor_id(): + """Current distribution LSB `Distributor ID`. + + :returns: + string value + + """ + if _distributor_id is None: + _init() + return _distributor_id + + +def release(): + """Current distribution LSB `Release`. + + :returns: + string value + + """ + if _release is None: + _init() + return _release + + +def _init(): + global _distributor_id, _release + + def check_derivates(): + global _distributor_id, _release + + if _distributor_id not in _DERIVATES: + return + _distributor_id, releases = _DERIVATES[_distributor_id] + for i in releases: + release_value = i(_release) + if release_value: + break + else: + release_value = '' + _release = release_value + + try: + _distributor_id, _release = _lsb_release() + check_derivates() + except OSError: + if exists('/etc/lsb-release'): + _distributor_id, _release = _parse_lsb_release() + check_derivates() + if not _release: + _distributor_id, _release = _find_lsb_release() + + return _distributor_id, _release + + +def _lsb_release(): + lsb_id, lsb_release = '', '' + + process = subprocess.Popen(['lsb_release', '--all'], + stderr=subprocess.PIPE, stdout=subprocess.PIPE) + stdout, __ = process.communicate() + + if process.returncode == 0: + for line in str(stdout).split('\n'): + if ':' not in line: + continue + key, value = line.split(':', 1) + if key.strip() == 'Distributor ID': + lsb_id = value.strip() + elif key.strip() == 'Release': + lsb_release = value.strip() + + return lsb_id, lsb_release + + +def _parse_lsb_release(): + lsb_id, lsb_release = '', '' + + for line in file('/etc/lsb-release').readlines(): + key, value = line.split('=') + value = value.strip().strip('\'"') + if key == 'DISTRIB_ID': + lsb_id = value + elif key == 'DISTRIB_RELEASE': + lsb_release = value + + return lsb_id, lsb_release + + +def _find_lsb_release(): + if exists('/etc/debian_version'): + return 'Debian', file('/etc/debian_version').read().strip() + + elif exists('/etc/redhat-release'): + line = file('/etc/redhat-release').read().strip() + + match = re.search('Fedora.*?\W([0-9.]+)', line) + if match is not None: + return 'Fedora', match.group(1) + + match = re.search('CentOS.*?\W([0-9.]+)', line) + if match is not None: + return 'CentOS', match.group(1) + + match = re.search('\W([0-9.]+)', line) + if match is not None: + return 'RHEL', match.group(1) + + else: + # TODO http://linuxmafia.com/faq/Admin/release-files.html + return '', '' + + +if __name__ == '__main__': + print distributor_id(), release() diff --git a/sugar_network/zerosugar/packagekit.py b/sugar_network/zerosugar/packagekit.py new file mode 100644 index 0000000..1434d4c --- /dev/null +++ b/sugar_network/zerosugar/packagekit.py @@ -0,0 +1,262 @@ +# Copyright (C) 2010-2012 Aleksey Lim +# Copyright (C) 2010 Thomas Leonard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import locale +import logging +from ConfigParser import ConfigParser +from os.path import exists +from gettext import gettext as _ + +import dbus +import gobject +from dbus.mainloop.glib import DBusGMainLoop + +from zeroinstall.injector import distro + +from sugar_network.zerosugar import pipe +from active_toolkit import enforce + + +_PK_CONFILE = '/etc/PackageKit/PackageKit.conf' + +_logger = logging.getLogger('zerosugar.packagekit') + +_pk = None +_pk_max_resolve = 100 +_pk_max_install = 2500 + +DBusGMainLoop(set_as_default=True) + + +def resolve(names): + enforce(_get_pk() is not None, 'Cannot connect to PackageKit') + + pipe.progress('resolve', + message=_('Resolving %s package name(s)') % len(names)) + _logger.debug('Resolve names %r', names) + result = {} + + mainloop = gobject.MainLoop() + while names: + chunk = names[:min(len(names), _pk_max_resolve)] + del names[:len(chunk)] + + transaction = _Transaction(mainloop.quit) + transaction.resolve(chunk) + mainloop.run() + + missed = set(chunk) - set([i['name'] for i in transaction.packages]) + enforce(not missed, + 'Failed to resolve %s package(s)', ', '.join(missed)) + for pkg in transaction.packages: + result[pkg['name']] = pkg + + return result + + +def install(packages): + ids = [i['pk_id'] for i in packages] + + pipe.progress('install', + message=_('Installing %s package(s)') % len(packages)) + _logger.debug('Ask PackageKit to install %r packages', ids) + + mainloop = gobject.MainLoop() + while ids: + chunk = ids[:min(len(ids), _pk_max_install)] + del ids[:len(chunk)] + + transaction = _Transaction(mainloop.quit) + transaction.install(chunk) + mainloop.run() + + enforce(transaction.error_code is None or + transaction.error_code in ('package-already-installed', + 'all-packages-already-installed'), + 'PackageKit install failed: %s (%s)', + transaction.error_details, transaction.error_code) + + +class _Transaction(object): + + def __init__(self, finished_cb): + self._finished_cb = finished_cb + self.error_code = None + self.error_details = None + self.packages = [] + + self._object = dbus.SystemBus().get_object( + # pylint: disable-msg=E1103 + 'org.freedesktop.PackageKit', _get_pk().GetTid(), False) + self._proxy = dbus.Interface(self._object, + 'org.freedesktop.PackageKit.Transaction') + self._props = dbus.Interface(self._object, dbus.PROPERTIES_IFACE) + + self._signals = [] + for signal, cb in [ + ('Finished', self.__finished_cb), + ('ErrorCode', self.__error_code_cb), + ('Package', self.__package_cb), + ]: + self._signals.append(self._proxy.connect_to_signal(signal, cb)) + + defaultlocale = locale.getdefaultlocale()[0] + if defaultlocale is not None: + self._compat_call([ + ('SetLocale', defaultlocale), + ('SetHints', ['locale=%s' % defaultlocale]), + ]) + + def resolve(self, names): + self._proxy.Resolve('none', names) + + def install(self, names): + _auth_wrapper('org.freedesktop.packagekit.package-install', + self._compat_call, [ + ('InstallPackages', names), + ('InstallPackages', True, names), + ]) + + def get_percentage(self): + if self._object is None: + return None + try: + return self._props.Get('org.freedesktop.PackageKit.Transaction', + 'Percentage') + except Exception: + result, __, __, __ = self._proxy.GetProgress() + return result + + def _compat_call(self, calls): + for call in calls: + method = call[0] + args = call[1:] + try: + dbus_method = self._proxy.get_dbus_method(method) + return dbus_method(*args) + except dbus.exceptions.DBusException, e: + if e.get_dbus_name() not in [ + 'org.freedesktop.DBus.Error.UnknownMethod', + 'org.freedesktop.DBus.Error.InvalidArgs']: + raise + raise Exception('Cannot call %r DBus method' % calls) + + def __finished_cb(self, status, runtime): + _logger.debug('Transaction finished: %s', status) + for i in self._signals: + i.remove() + self._finished_cb() + self._props = None + self._proxy = None + self._object = None + + def __error_code_cb(self, code, details): + self.error_code = code + self.error_details = details + + def __package_cb(self, status, pk_id, summary): + package_name, version, arch, __ = pk_id.split(';') + clean_version = distro.try_cleanup_distro_version(version) + if not clean_version: + _logger.warn('Cannot parse distribution version "%s" ' + 'for package "%s"', version, package_name) + package = { + 'pk_id': str(pk_id), + 'version': clean_version, + 'name': package_name, + 'arch': distro.canonical_machine(arch), + 'installed': (status == 'installed'), + } + _logger.debug('Resolved PackageKit name: %r', package) + self.packages.append(package) + + +def _get_pk(): + global _pk, _pk_max_resolve, _pk_max_install + + if _pk is not None: + if _pk is False: + return None + else: + return _pk + + try: + bus = dbus.SystemBus() + pk_object = bus.get_object('org.freedesktop.PackageKit', + '/org/freedesktop/PackageKit', False) + _pk = dbus.Interface(pk_object, 'org.freedesktop.PackageKit') + _logger.info('PackageKit dbus service found') + except Exception, error: + _pk = False + _logger.info('PackageKit dbus service not found: %s', error) + return None + + if exists(_PK_CONFILE): + conf = ConfigParser() + conf.read(_PK_CONFILE) + if conf.has_option('Daemon', 'MaximumItemsToResolve'): + _pk_max_resolve = \ + int(conf.get('Daemon', 'MaximumItemsToResolve')) + if conf.has_option('Daemon', 'MaximumPackagesToProcess'): + _pk_max_install = \ + int(conf.get('Daemon', 'MaximumPackagesToProcess')) + + return _pk + + +def _auth_wrapper(iface, method, *args): + _logger.info('Obtain authentication for %s', iface) + + def obtain(): + pk_auth = dbus.SessionBus().get_object( + 'org.freedesktop.PolicyKit.AuthenticationAgent', '/', + 'org.freedesktop.PolicyKit.AuthenticationAgent') + pk_auth.ObtainAuthorization(iface, dbus.UInt32(0), + dbus.UInt32(os.getpid()), timeout=300) + + try: + # PK on f11 needs to obtain authentication at first + obtain() + return method(*args) + except Exception: + # It seems doesn't work for recent PK + try: + return method(*args) + except dbus.exceptions.DBusException, e: + if e.get_dbus_name() != \ + 'org.freedesktop.PackageKit.Transaction.RefusedByPolicy': + raise + iface, auth = e.get_dbus_message().split() + if not auth.startswith('auth_'): + raise + obtain() + return method(*args) + + +if __name__ == '__main__': + import sys + from pprint import pprint + + if len(sys.argv) == 1: + exit() + + logging.basicConfig(level=logging.DEBUG) + + if sys.argv[1] == 'install': + install(resolve(sys.argv[2:]).values()) + else: + pprint(resolve(sys.argv[1:])) diff --git a/sugar_network/zerosugar/pipe.py b/sugar_network/zerosugar/pipe.py new file mode 100644 index 0000000..0174ca4 --- /dev/null +++ b/sugar_network/zerosugar/pipe.py @@ -0,0 +1,185 @@ +# Copyright (C) 2010-2012 Aleksey Lim +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import sys +import json +import signal +import logging +import threading +from os.path import exists + +from sugar_network import IPCClient +from sugar_network.zerosugar import feeds +from sugar_network.toolkit import sugar +from active_toolkit import coroutine, util + + +_logger = logging.getLogger('zerosugar.pipe') +_pipe = None + + +def progress(state, **event): + if _pipe is None: + return + event['state'] = state + os.write(_pipe, json.dumps(event)) + os.write(_pipe, '\n') + + +def fork(callback, mountpoint, context, *args): + fd_r, fd_w = os.pipe() + + pid = os.fork() + if pid: + os.close(fd_w) + return _Pipe(pid, fd_r) + + os.close(fd_r) + global _pipe + _pipe = fd_w + + def thread_func(): + progress(state='boot', + session={ + 'log_path': _setup_logging(context), + 'mountpoint': mountpoint, + 'context': context, + }) + + # TODO Stub + feeds.clients.append(IPCClient(mountpoint=mountpoint)) + if mountpoint != '~': + feeds.clients.append(IPCClient(mountpoint='~')) + if mountpoint != '/': + feeds.clients.append(IPCClient(mountpoint='/')) + + try: + callback(mountpoint, context, *args) + except Exception, error: + util.exception(_logger) + progress(state='failure', error=str(error)) + + # Avoid a mess with current thread coroutines + thread = threading.Thread(target=thread_func) + thread.start() + thread.join() + + os.close(fd_w) + sys.stdout.flush() + sys.stderr.flush() + # pylint: disable-msg=W0212 + os._exit(0) + + +class _Pipe(object): + + def __init__(self, pid, fd): + self._pid = pid + self._file = os.fdopen(fd) + self._session = {} + + def fileno(self): + return None if self._file is None else self._file.fileno() + + def read(self): + if self._file is None: + return None + + event = self._file.readline() + if not event: + status = 0 + try: + __, status = os.waitpid(self._pid, 0) + except OSError: + pass + failure = _decode_exit_failure(status) + if failure: + event = {'state': 'failure', 'error': failure} + event.update(self._session) + return event + else: + self._file.close() + self._file = None + return None + + event = json.loads(event) + if 'session' in event: + self._session.update(event.pop('session')) + event.update(self._session) + return event + + def __iter__(self): + if self._file is None: + return + while True: + coroutine.select([self._file.fileno()], [], []) + event = self.read() + if event is None: + break + yield event + + +def _decode_exit_failure(status): + failure = None + if os.WIFEXITED(status): + status = os.WEXITSTATUS(status) + if status: + failure = 'Exited with status %s' % status + elif os.WIFSIGNALED(status): + signum = os.WTERMSIG(status) + if signum not in (signal.SIGINT, signal.SIGKILL, signal.SIGTERM): + failure = 'Terminated by signal %s' % signum + else: + signum = os.WTERMSIG(status) + failure = 'Undefined status with signal %s' % signum + return failure + + +def _setup_logging(context): + log_dir = sugar.profile_path('logs') + if not exists(log_dir): + os.makedirs(log_dir) + path = util.unique_filename(log_dir, context + '.log') + + def stdfd(stream): + # pylint: disable-msg=W0212 + + if hasattr(stream, 'fileno'): + return stream.fileno() + else: + # Sugar Shell wraps std streams + return stream._stream.fileno() + + logfile = file(path, 'a+') + os.dup2(logfile.fileno(), stdfd(sys.stdout)) + os.dup2(logfile.fileno(), stdfd(sys.stderr)) + logfile.close() + + debug = sugar.logger_level() + if not debug: + level = logging.WARNING + elif debug == 1: + level = logging.INFO + else: + level = logging.DEBUG + + root_logger = logging.getLogger() + for handler in root_logger.handlers: + root_logger.removeHandler(handler) + logging.basicConfig(level=level, + format='%(asctime)s %(levelname)s %(name)s: %(message)s') + + return path diff --git a/sugar_network/zerosugar/solution.py b/sugar_network/zerosugar/solution.py index 98a36c8..9bc948d 100644 --- a/sugar_network/zerosugar/solution.py +++ b/sugar_network/zerosugar/solution.py @@ -13,23 +13,74 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -# pylint: disable-msg=E1101,E0102 +import logging + +from zeroinstall.injector.config import Config +from zeroinstall.injector.driver import Driver + +from sugar_network.zerosugar import packagekit +from active_toolkit import enforce DEEP = 2 +_logger = logging.getLogger('zerosugar.solution') + + +def solve(req): + config = Config() + driver = Driver(config, req) + driver.solver.record_details = True + + while True: + driver.solver.solve(req.interface_uri, + driver.target_arch, command_name=req.command) + if driver.solver.ready: + break + + missed = [] + packaged_feeds = [] + to_resolve = [] + + for url in driver.solver.feeds_used: + feed = config.iface_cache.get_feed(url) + if feed is None: + missed.append(url) + elif feed.to_resolve: + packaged_feeds.append(feed) + to_resolve.extend(feed.to_resolve) + + enforce(not missed, 'Cannot find feed(s) for %s', ', '.join(missed)) + if not to_resolve: + break + + resolved = packagekit.resolve(to_resolve) + for feed in packaged_feeds: + feed.resolve([resolved[i] for i in feed.to_resolve]) + + _logger.debug('\n'.join( + ['Solve results:'] + + [' %s: %s' % (k.uri, v) for k, v in driver.solver.details.items()])) + + if not driver.solver.ready: + # pylint: disable-msg=W0212 + reason = driver.solver._failure_reason + if not reason: + missed = [iface.uri for iface, impl in + driver.solver.selections.items() if impl is None] + reason = 'Cannot find implementations for %s' % ', '.join(missed) + raise RuntimeError(reason) + + return Solution(driver.solver.selections) + class Solution(object): interface = property(lambda self: self.value.interface) selections = property(lambda self: self.value.selections) - def __init__(self, value, req): + def __init__(self, value): self.value = value - self.ready = True - self.details = {} - self.failure_reason = None - self.requirements = req def __getitem__(self, url): selection = self.selections.get(url) diff --git a/tests/units/home_mount.py b/tests/units/home_mount.py index c0cfaae..dc339d5 100755 --- a/tests/units/home_mount.py +++ b/tests/units/home_mount.py @@ -2,6 +2,7 @@ # sugar-lint: disable import os +import json import socket from os.path import exists, abspath @@ -9,6 +10,7 @@ from __init__ import tests from active_toolkit import sockets, coroutine from sugar_network.resources.report import Report +from sugar_network.local import activities from sugar_network import IPCClient @@ -171,6 +173,69 @@ class HomeMountTest(tests.Test): ], events) + def test_Feed(self): + self.touch(('Activities/activity-1/activity/activity.info', [ + '[Activity]', + 'name = TestActivity', + 'bundle_id = bundle_id', + 'exec = false', + 'icon = icon', + 'activity_version = 1', + 'license = Public Domain', + ])) + self.touch(('Activities/activity-2/activity/activity.info', [ + '[Activity]', + 'name = TestActivity', + 'bundle_id = bundle_id', + 'exec = true', + 'icon = icon', + 'activity_version = 2', + 'license = Public Domain', + 'requires = dep1; dep2 = 1; dep3 < 2; dep4 >= 3', + ])) + + self.start_server() + client = IPCClient(mountpoint='~') + + monitor = coroutine.spawn(activities.monitor, + self.mounts.volume['context'], ['Activities']) + coroutine.sleep() + + self.assertEqual({ + 'versions': { + '1': { + '*-*': { + 'commands': { + 'activity': { + 'exec': 'false', + }, + }, + 'stability': 'stable', + 'guid': tests.tmpdir + '/Activities/activity-1', + 'requires': {}, + }, + }, + '2': { + '*-*': { + 'commands': { + 'activity': { + 'exec': 'true', + }, + }, + 'stability': 'stable', + 'guid': tests.tmpdir + '/Activities/activity-2', + 'requires': { + 'dep1': {}, + 'dep2': {'restrictions': [['1', '2']]}, + 'dep3': {'restrictions': [[None, '2']]}, + 'dep4': {'restrictions': [['3', None]]}, + }, + }, + }, + }, + }, + client.get(['context', 'bundle_id', 'feed'], cmd='get_blob')) + if __name__ == '__main__': tests.main() diff --git a/tests/units/injector.py b/tests/units/injector.py index 43da2c2..29acf96 100755 --- a/tests/units/injector.py +++ b/tests/units/injector.py @@ -15,6 +15,7 @@ from sugar_network import checkin, launch from sugar_network.resources.user import User from sugar_network.resources.context import Context from sugar_network.resources.implementation import Implementation +from sugar_network.zerosugar import lsb_release, packagekit from sugar_network.local import activities from sugar_network import IPCClient @@ -60,16 +61,18 @@ class InjectorTest(tests.Test): self.touch( (blob_path, '{}'), (blob_path + '.blob', json.dumps({ - '1': { - '*-*': { - 'commands': { - 'activity': { - 'exec': 'echo', + 'versions': { + '1': { + '*-*': { + 'commands': { + 'activity': { + 'exec': 'echo', + }, }, + 'stability': 'stable', + 'guid': impl, + 'size': 0, }, - 'stability': 'stable', - 'guid': impl, - 'size': 0, }, }, })), @@ -129,17 +132,19 @@ class InjectorTest(tests.Test): self.touch( (blob_path, '{}'), (blob_path + '.blob', json.dumps({ - '1': { - '*-*': { - 'commands': { - 'activity': { - 'exec': 'false', + 'versions': { + '1': { + '*-*': { + 'commands': { + 'activity': { + 'exec': 'false', + }, }, + 'stability': 'stable', + 'guid': impl, + 'size': 0, + 'extract': 'TestActivitry', }, - 'stability': 'stable', - 'guid': impl, - 'size': 0, - 'extract': 'TestActivitry', }, }, })), @@ -186,30 +191,32 @@ class InjectorTest(tests.Test): self.touch( (blob_path, '{}'), (blob_path + '.blob', json.dumps({ - '1': { - '*-*': { - 'commands': { - 'activity': { - 'exec': 'false', + 'versions': { + '1': { + '*-*': { + 'commands': { + 'activity': { + 'exec': 'false', + }, }, + 'stability': 'stable', + 'guid': impl, + 'size': 0, + 'extract': 'TestActivitry', }, - 'stability': 'stable', - 'guid': impl, - 'size': 0, - 'extract': 'TestActivitry', }, - }, - '2': { - '*-*': { - 'commands': { - 'activity': { - 'exec': 'true', + '2': { + '*-*': { + 'commands': { + 'activity': { + 'exec': 'true', + }, }, + 'stability': 'stable', + 'guid': impl_2, + 'size': 0, + 'extract': 'TestActivitry', }, - 'stability': 'stable', - 'guid': impl_2, - 'size': 0, - 'extract': 'TestActivitry', }, }, })), @@ -240,60 +247,122 @@ class InjectorTest(tests.Test): ], [i for i in pipe]) - def test_OfflineFeed(self): - self.touch(('Activities/activity-1/activity/activity.info', [ + def test_MissedFeeds(self): + self.start_server() + + context = 'fake' + pipe = launch('~', context) + log_path = tests.tmpdir + '/.sugar/default/logs/%s.log' % context + self.assertEqual([ + {'state': 'boot', 'mountpoint': '~', 'context': context, 'log_path': log_path}, + {'state': 'analyze', 'mountpoint': '~', 'context': context, 'log_path': log_path}, + {'state': 'failure', 'error': 'Cannot find feed(s) for %s' % context, 'mountpoint': '~', 'context': context, 'log_path': log_path}, + ], + [i for i in pipe]) + + def test_launch_Offline(self): + self.touch(('Activities/activity/activity/activity.info', [ '[Activity]', 'name = TestActivity', 'bundle_id = bundle_id', 'exec = false', 'icon = icon', 'activity_version = 1', - 'license=Public Domain', + 'license = Public Domain', ])) - self.touch(('Activities/activity-2/activity/activity.info', [ + + self.start_server() + monitor = coroutine.spawn(activities.monitor, + self.mounts.volume['context'], ['Activities']) + coroutine.sleep() + + context = 'bundle_id' + impl = tests.tmpdir + '/Activities/activity' + + pipe = launch('~', context) + log_path = tests.tmpdir + '/.sugar/default/logs/%s.log' % context + self.assertEqual([ + {'state': 'boot', 'mountpoint': '~', 'context': context, 'log_path': log_path}, + {'state': 'analyze', 'mountpoint': '~', 'context': context, 'log_path': log_path}, + {'state': 'ready', 'implementation': impl, 'mountpoint': '~', 'context': context, 'log_path': log_path}, + {'state': 'exec', 'implementation': impl, 'mountpoint': '~', 'context': context, 'log_path': log_path}, + ], + [i for i in pipe]) + + def test_InstallDeps(self): + self.touch(('Activities/activity/activity/activity.info', [ '[Activity]', 'name = TestActivity', 'bundle_id = bundle_id', 'exec = true', 'icon = icon', - 'activity_version = 2', - 'license=Public Domain', + 'activity_version = 1', + 'license = Public Domain', + 'requires = dep1; dep2', ])) - self.start_server() - client = IPCClient(mountpoint='~') - + self.touch('remote/master') + self.start_ipc_and_restful_server([User, Context, Implementation]) + remote = IPCClient(mountpoint='/') monitor = coroutine.spawn(activities.monitor, self.mounts.volume['context'], ['Activities']) coroutine.sleep() - blob = client.get(['context', 'bundle_id', 'feed'], cmd='get_blob') - self.assertEqual( - json.dumps({ - '1': { - '*-*': { - 'commands': { - 'activity': { - 'exec': 'false', - }, - }, - 'stability': 'stable', - 'guid': tests.tmpdir + '/Activities/activity-1', + remote.post(['context'], { + 'type': 'package', + 'title': 'title', + 'summary': 'summary', + 'description': 'description', + 'implement': 'dep1', + }) + blob_path = 'remote/context/de/dep1/feed' + self.touch( + (blob_path, '{}'), + (blob_path + '.blob', json.dumps({ + 'packages': { + lsb_release.distributor_id(): { + 'binary': ['dep1.bin'], }, }, - '2': { - '*-*': { - 'commands': { - 'activity': { - 'exec': 'true', - }, - }, - 'stability': 'stable', - 'guid': tests.tmpdir + '/Activities/activity-2', + })), + ) + + remote.post(['context'], { + 'type': 'package', + 'title': 'title', + 'summary': 'summary', + 'description': 'description', + 'implement': 'dep2', + }) + blob_path = 'remote/context/de/dep2/feed' + self.touch( + (blob_path, '{}'), + (blob_path + '.blob', json.dumps({ + 'packages': { + lsb_release.distributor_id(): { + 'binary': ['dep2.bin'], }, }, - }), - blob) + })), + ) + + def resolve(names): + with file('resolve', 'w') as f: + json.dump(names, f) + return dict([(i, {'name': i, 'pk_id': i, 'version': '0', 'arch': '*', 'installed': i == 'dep1.bin'}) for i in names]) + + def install(packages): + with file('install', 'w') as f: + json.dump([i['name'] for i in packages], f) + + self.override(packagekit, 'resolve', resolve) + self.override(packagekit, 'install', install) + + context = 'bundle_id' + pipe = launch('~', context) + self.assertEqual('exec', [i for i in pipe][-1].get('state')) + self.assertEqual(['dep1.bin', 'dep2.bin'], json.load(file('resolve'))) + self.assertEqual(['dep2.bin'], json.load(file('install'))) if __name__ == '__main__': diff --git a/tests/units/node.py b/tests/units/node.py index 51882e8..d0ff38f 100755 --- a/tests/units/node.py +++ b/tests/units/node.py @@ -342,6 +342,32 @@ class NodeTest(tests.Test): self.assertRaises(ad.NotFound, call, cp, method='GET', document='context', guid=guid) self.assertEqual([], call(cp, method='GET', document='context')['result']) + def test_SetGuidOnMaster(self): + volume1 = Volume('db1') + cp1 = NodeCommands(volume1) + call(cp1, method='POST', document='context', principal='principal', content={ + 'type': 'activity', + 'title': 'title', + 'summary': 'summary', + 'description': 'description', + 'implement': 'foo', + }) + self.assertRaises(ad.NotFound, call, cp1, method='GET', document='context', guid='foo') + + volume2 = Volume('db2') + self.touch('db2/master') + cp2 = NodeCommands(volume2) + call(cp2, method='POST', document='context', principal='principal', content={ + 'type': 'activity', + 'title': 'title', + 'summary': 'summary', + 'description': 'description', + 'implement': 'foo', + }) + self.assertEqual( + {'guid': 'foo', 'implement': ['foo'], 'title': 'title'}, + call(cp2, method='GET', document='context', guid='foo', reply=['guid', 'implement', 'title'])) + def call(cp, principal=None, content=None, **kwargs): request = ad.Request(**kwargs) diff --git a/tests/units/node_mount.py b/tests/units/node_mount.py index 0a67bc2..763a73d 100755 --- a/tests/units/node_mount.py +++ b/tests/units/node_mount.py @@ -194,17 +194,19 @@ class NodeMountTest(tests.Test): }, f) with file('mnt/context/%s/%s/feed.blob' % (context[:2], context), 'w') as f: json.dump({ - '1': { - '*-*': { - 'commands': { - 'activity': { - 'exec': 'echo', + 'versions': { + '1': { + '*-*': { + 'commands': { + 'activity': { + 'exec': 'echo', + }, }, + 'stability': 'stable', + 'guid': impl, + 'size': 0, + 'extract': 'TestActivitry', }, - 'stability': 'stable', - 'guid': impl, - 'size': 0, - 'extract': 'TestActivitry', }, }, }, f) |