From 89057aecc05219179107351bd2e8b3ecbc59d27d Mon Sep 17 00:00:00 2001 From: Aleksey Lim Date: Tue, 08 Oct 2013 04:12:28 +0000 Subject: Do not recycle running implementations --- diff --git a/TODO b/TODO index 94f2e8c..0f53069 100644 --- a/TODO +++ b/TODO @@ -13,3 +13,4 @@ - slave._Pooler might leak events if pullers are not in time to call wait() - revert per-document "downloads" property as "launches", a part of unpersonizalied user_stats - sync node->local db sync +- parse command while uploading impls; while parsing, take into accoun quotes diff --git a/sugar_network/client/cache.py b/sugar_network/client/cache.py index 988f789..26b940d 100644 --- a/sugar_network/client/cache.py +++ b/sugar_network/client/cache.py @@ -34,11 +34,16 @@ class Cache(object): self._volume = volume self._pool = None self._du = 0 + self._acquired = {} def __iter__(self): self._ensure_open() return iter(self._pool) + @property + def du(self): + return self._du + def ensure(self, requested_size, temp_size=0): self._ensure_open() to_free = self._to_free(requested_size, temp_size) @@ -51,26 +56,40 @@ class Cache(object): if to_free <= 0: break - def checkin(self, guid): + def acquire(self, guid, size): + self.checkout(guid) + self._acquired.setdefault(guid, [0, size])[0] += 1 + return guid + + def release(self, *guids): + for guid in guids: + acquired = self._acquired.get(guid) + if acquired is None: + continue + acquired[0] -= 1 + if acquired[0] <= 0: + self.checkin(guid, acquired[1]) + del self._acquired[guid] + + def checkin(self, guid, size): self._ensure_open() if guid in self._pool: self._pool.__getitem__(guid) return - _logger.debug('Checkin %r', guid) - impls = self._volume['implementation'] - meta = impls.get(guid).meta('data') - size = meta.get('unpack_size') or meta['blob_size'] - mtime = os.stat(impls.path(guid)).st_mtime + _logger.debug('Checkin %r %d bytes long', guid, size) + mtime = os.stat(self._volume['implementation'].path(guid)).st_mtime self._pool[guid] = (size, mtime) + self._du += size - def checkout(self, guid): - self._ensure_open() + def checkout(self, guid, *args): if guid not in self._pool: - return + return False + self._ensure_open() _logger.debug('Checkout %r', guid) size, __ = self._pool.peek(guid) self._du -= size del self._pool[guid] + return True def recycle(self): self._ensure_open() @@ -93,13 +112,12 @@ class Cache(object): _logger.debug('Open implementations pool') pool = [] - contexts = self._volume['context'] impls = self._volume['implementation'] for res in impls.find(not_layer=['local'])[0]: meta = res.meta('data') if not meta or 'blob_size' not in meta: continue - clone = contexts.path(res['context'], '.clone') + clone = self._volume['context'].path(res['context'], '.clone') if exists(clone) and basename(os.readlink(clone)) == res.guid: continue pool.append(( diff --git a/sugar_network/client/implementations.py b/sugar_network/client/implementations.py index 5ab2e5b..841fb5f 100644 --- a/sugar_network/client/implementations.py +++ b/sugar_network/client/implementations.py @@ -62,7 +62,7 @@ class Routes(object): @route('GET', ['context', None], cmd='launch', arguments={'args': list}, mime_type='text/event-stream') - def launch(self, request, no_spawn): + def launch(self, request): activity_id = request.get('activity_id') if 'object_id' in request and not activity_id: activity_id = journal.get(request['object_id'], 'activity_id') @@ -73,22 +73,28 @@ class Routes(object): for context in self._checkin_context(request): yield {'event': 'launch', 'activity_id': activity_id}, request - impl = self._solve_impl(context, request) - if 'activity' not in context['type']: - app = request.get('context') or \ - _mimetype_context(impl['mime_type']) - enforce(app, 'Cannot find proper application') - self._checkin_impl(context, request, impl) - request = Request(path=['context', app], - object_id=impl['path'], session=request.session) - for context in self._checkin_context(request): - impl = self._solve_impl(context, request) - self._checkin_impl(context, request, impl) - - child = _exec(context, request, impl) - yield {'event': 'exec', 'activity_id': activity_id} - - status = child.wait() + acquired = [] + try: + impl = self._solve_impl(context, request) + if 'activity' not in context['type']: + app = request.get('context') or \ + _mimetype_context(impl['data']['mime_type']) + enforce(app, 'Cannot find proper application') + acquired += self._checkin_impl( + context, request, self._cache.acquire) + request = Request(path=['context', app], + object_id=impl['path'], session=request.session) + for context in self._checkin_context(request): + impl = self._solve_impl(context, request) + acquired += self._checkin_impl( + context, request, self._cache.acquire) + + child = _exec(context, request, impl) + yield {'event': 'exec', 'activity_id': activity_id} + status = child.wait() + finally: + self._cache.release(*acquired) + _logger.debug('Exit %s[%s]: %r', context.guid, child.pid, status) enforce(status == 0, 'Process exited with %r status', status) yield {'event': 'exit', 'activity_id': activity_id} @@ -102,14 +108,16 @@ class Routes(object): cloned_path = context.path('.clone') if request.content: impl = self._solve_impl(context, request) - self._checkin_impl(context, request, impl) + self._checkin_impl(context, request, self._cache.checkout) impl_path = relpath(dirname(impl['path']), context.path()) os.symlink(impl_path, cloned_path) - self._cache.checkout(impl['guid']) yield {'event': 'ready'} else: cloned_impl = basename(os.readlink(cloned_path)) - self._cache.checkin(cloned_impl) + meta = self._volume['implementation'].get( + cloned_impl).meta('data') + size = meta.get('unpack_size') or meta['blob_size'] + self._cache.checkin(cloned_impl, size) os.unlink(cloned_path) @route('GET', ['context', None], cmd='clone', @@ -140,11 +148,10 @@ class Routes(object): raise http.ServiceUnavailable, error, sys.exc_info()[2] def _checkin_context(self, request, layer=None): + contexts = self._volume['context'] guid = request.guid - if layer and not request.content and \ - not self._volume['context'].exists(guid): + if layer and not request.content and not contexts.exists(guid): return - contexts = self._volume['context'] if not contexts.exists(guid): context = self._call(method='GET', path=['context', guid]) @@ -184,47 +191,95 @@ class Routes(object): _logger.debug('Solving %r stability=%r', request.guid, stability) - if 'activity' not in context['type']: - _logger.debug('Cloniing %r', request.guid) - response = Response() - blob = self._call(method='GET', path=['context', request.guid], - cmd='clone', stability=stability, response=response) - response.meta['data']['blob'] = blob - return response.meta - solution, stale = self._cache_solution_get(request.guid, stability) if stale is False: _logger.debug('Reuse cached %r solution', request.guid) elif solution is not None and not self.inline(): - _logger.debug('Reuse stale %r solution in offline', request.guid) - else: - _logger.debug('Solve %r', request.guid) + _logger.debug('Reuse stale %r in offline', request.guid) + elif 'activity' in context['type']: from sugar_network.client import solver solution = self._map_exceptions(solver.solve, self.fallback, request.guid, stability) + else: + response = Response() + blob = self._call(method='GET', path=['context', request.guid], + cmd='clone', stability=stability, response=response) + response.meta['data']['blob'] = blob + solution = [response.meta] + request.session['solution'] = solution return solution[0] - def _checkin_impl(self, context, request, sel): - if 'activity' not in context['type']: - self._cache_impl(context, sel) - return + def _checkin_impl(self, context, request, cache_call): + if 'clone' in context['layer']: + cache_call = self._cache.checkout + impls = self._volume['implementation'] - to_install = [] - for sel in request.session['solution']: - if 'install' in sel: - enforce(self.inline(), http.ServiceUnavailable, - 'Installation is not available in offline') - to_install.extend(sel.pop('install')) - if to_install: - packagekit.install(to_install) + if 'activity' in context['type']: + to_install = [] + for sel in request.session['solution']: + if 'install' in sel: + enforce(self.inline(), http.ServiceUnavailable, + 'Installation is not available in offline') + to_install.extend(sel.pop('install')) + if to_install: + packagekit.install(to_install) + + def cache_impl(sel): + guid = sel['guid'] + data = sel['data'] + data_path = sel['path'] = impls.path(guid, 'data') + size = data.get('unpack_size') or data['blob_size'] + + blob = None + if 'blob' in data: + blob = data.pop('blob') + + if impls.exists(guid): + return cache_call(guid, size) + + if blob is None: + blob = self._call(method='GET', + path=['implementation', guid, 'data']) + try: + if not exists(dirname(data_path)): + os.makedirs(dirname(data_path)) + if 'activity' in context['type']: + self._cache.ensure(size, data['blob_size']) + with toolkit.TemporaryFile() as tmp_file: + shutil.copyfileobj(blob, tmp_file) + tmp_file.seek(0) + with Bundle(tmp_file, 'application/zip') as bundle: + bundle.extractall(data_path, prefix=bundle.rootdir) + for exec_dir in ('bin', 'activity'): + bin_path = join(data_path, exec_dir) + if not exists(bin_path): + continue + for filename in os.listdir(bin_path): + os.chmod(join(bin_path, filename), 0755) + else: + self._cache.ensure(size) + with file(data_path, 'wb') as f: + shutil.copyfileobj(blob, f) + impl = sel.copy() + impl['layer'] = [] + impl['ctime'] = impl['mtime'] = int(time.time()) + impl['author'] = {} + impl['notes'] = '' + impl['tags'] = [] + impls.create(impl) + return cache_call(guid, size) + except Exception: + shutil.rmtree(data_path, ignore_errors=True) + raise + result = [] for sel in request.session['solution']: if 'path' not in sel and sel['stability'] != 'packaged': - self._cache_impl(context, sel) - - self._cache_solution(context.guid, + result.append(cache_impl(sel)) + self._cache_solution_set(context.guid, request.session['stability'], request.session['solution']) + return result def _cache_solution_path(self, guid): return client.path('solutions', guid[:2], guid) @@ -248,70 +303,17 @@ class Routes(object): stale = (self._node_mtime > os.stat(path).st_mtime) if not stale: stale = (packagekit.mtime() > os.stat(path).st_mtime) + return _CachedSolution(solution), stale - return solution, stale - - def _cache_solution(self, guid, stability, solution): + def _cache_solution_set(self, guid, stability, solution): + if isinstance(solution, _CachedSolution): + return path = self._cache_solution_path(guid) if not exists(dirname(path)): os.makedirs(dirname(path)) with file(path, 'w') as f: json.dump([client.api_url.value, stability, solution], f) - def _cache_impl(self, context, sel): - guid = sel['guid'] - impls = self._volume['implementation'] - data_path = sel['path'] = impls.path(guid, 'data') - - if impls.exists(guid): - self._cache.checkin(guid) - return - - if 'data' in sel: - data = sel.pop('data') - blob = data.pop('blob') - else: - response = Response() - blob = self._call(method='GET', - path=['implementation', guid, 'data'], response=response) - data = response.meta - for key in ('seqno', 'url'): - if key in data: - del data[key] - - try: - if not exists(dirname(data_path)): - os.makedirs(dirname(data_path)) - if 'activity' in context['type']: - self._cache.ensure(data['unpack_size'], data['blob_size']) - with toolkit.TemporaryFile() as tmp_file: - shutil.copyfileobj(blob, tmp_file) - tmp_file.seek(0) - with Bundle(tmp_file, 'application/zip') as bundle: - bundle.extractall(data_path, prefix=bundle.rootdir) - for exec_dir in ('bin', 'activity'): - bin_path = join(data_path, exec_dir) - if not exists(bin_path): - continue - for filename in os.listdir(bin_path): - os.chmod(join(bin_path, filename), 0755) - else: - self._cache.ensure(data['blob_size']) - with file(data_path, 'wb') as f: - shutil.copyfileobj(blob, f) - impl = sel.copy() - impl['data'] = data - impl['layer'] = [] - impl['ctime'] = impl['mtime'] = int(time.time()) - impl['author'] = {} - impl['notes'] = '' - impl['tags'] = [] - impls.create(impl) - self._cache.checkin(guid) - except Exception: - shutil.rmtree(data_path, ignore_errors=True) - raise - def _get_clone(self, request, response): for context in self._checkin_context(request): if 'clone' not in context['layer']: @@ -352,7 +354,8 @@ def _exec(context, request, sel): if not exists(path): os.makedirs(path) - args = sel['command'] + [ + cmd = sel['data']['spec']['*-*']['commands']['activity']['exec'] + args = cmd.split() + [ '-b', request.guid, '-a', request.session['activity_id'], ] @@ -404,3 +407,7 @@ def _exec(context, request, sel): logging.exception('Failed to execute %r args=%r', sel, args) finally: os._exit(1) + + +class _CachedSolution(list): + pass diff --git a/sugar_network/client/solver.py b/sugar_network/client/solver.py index 9f1b6e7..4d4e341 100644 --- a/sugar_network/client/solver.py +++ b/sugar_network/client/solver.py @@ -163,20 +163,12 @@ def _interface_init(self, url): def _impl_new(config, iface, sel): - impl = {'guid': sel.id, - 'context': iface, - 'license': sel.impl.license, - 'version': sel.version, - 'stability': sel.impl.upstream_stability.name, - } - + impl = sel.impl.sn_impl + impl['context'] = iface if sel.local_path: impl['path'] = sel.local_path if sel.impl.to_install: impl['install'] = sel.impl.to_install - commands = sel.get_commands() - if commands: - impl['command'] = commands.values()[0].path.split() return impl @@ -253,30 +245,38 @@ class _Feed(model.ZeroInstallFeed): impl.upstream_stability = model.stability_levels['packaged'] impl.to_install = [i for i in packages if not i['installed']] impl.add_download_source(self.context, 0, None) + impl.sn_impl = { + 'guid': self.context, + 'license': None, + 'version': top_package['version'], + 'stability': 'packaged', + } self.implementations[self.context] = impl def implement(self, release): impl_id = release['guid'] + spec = release['data']['spec']['*-*'] impl = _Implementation(self, impl_id, None) impl.version = parse_version(release['version']) impl.released = 0 - impl.arch = release['arch'] + impl.arch = '*-*' impl.upstream_stability = model.stability_levels['stable'] - impl.requires.extend(_read_requires(release.get('requires'))) - impl.hints = release impl.license = release.get('license') or [] + impl.requires = _read_requires(spec.get('requires')) + impl.requires.extend(_read_requires(release.get('requires'))) + impl.sn_impl = release if isabs(impl_id): impl.local_path = impl_id else: impl.add_download_source(impl_id, 0, None) - for name, command in release['commands'].items(): + for name, command in spec['commands'].items(): impl.commands[name] = _Command(name, command) - for name, insert, mode in release.get('bindings') or []: + for name, insert, mode in spec.get('bindings') or []: binding = model.EnvironmentBinding(name, insert, mode=mode) impl.bindings.append(binding) @@ -290,12 +290,18 @@ class _Feed(model.ZeroInstallFeed): impl.arch = '*-*' impl.upstream_stability = model.stability_levels['packaged'] self.implementations[impl_id] = impl + impl.sn_impl = { + 'guid': impl_id, + 'license': None, + 'version': sugar_version, + 'stability': 'packaged', + } class _Implementation(model.ZeroInstallImplementation): to_install = None - hints = None + sn_impl = None license = None def is_available(self, stores): @@ -341,15 +347,14 @@ class _Dependency(model.InterfaceDependency): class _Command(model.Command): - def __init__(self, name, data): + def __init__(self, name, command): self.qdom = None self.name = name - self._path = data['exec'] - self._requires = _read_requires(data.get('requires')) + self._requires = _read_requires(command.get('requires')) @property def path(self): - return self._path + return 'doesnt_matter' @property def requires(self): diff --git a/sugar_network/db/routes.py b/sugar_network/db/routes.py index 7a4b582..b863b64 100644 --- a/sugar_network/db/routes.py +++ b/sugar_network/db/routes.py @@ -242,11 +242,10 @@ class Routes(object): meta.pop('blob') else: meta = value - response.content_length = meta.get('blob_size') or 0 - response.meta.update(meta) - if 'mtime' in meta: - response.last_modified = meta['mtime'] + response.meta = meta + response.last_modified = meta.get('mtime') + response.content_length = meta.get('blob_size') or 0 return value diff --git a/sugar_network/model/routes.py b/sugar_network/model/routes.py index 628a057..d36e1c8 100644 --- a/sugar_network/model/routes.py +++ b/sugar_network/model/routes.py @@ -37,22 +37,17 @@ class VolumeRoutes(db.Routes): impls, __ = implementations.find(context=context.guid, not_layer='deleted', **request) for impl in impls: - for arch, spec in impl.meta('data')['spec'].items(): - spec['guid'] = impl.guid - spec['version'] = impl['version'] - spec['arch'] = arch - spec['stability'] = impl['stability'] - spec['license'] = impl['license'] - if context['dependencies']: - requires = spec.setdefault('requires', {}) - for i in context['dependencies']: - requires.setdefault(i, {}) - blob = implementations.get(impl.guid).meta('data') - if blob: - for key in ('blob_size', 'unpack_size'): - if key in blob: - spec[key] = blob[key] - versions.append(spec) + version = impl.properties( + ['guid', 'version', 'stability', 'license']) + if context['dependencies']: + requires = version.setdefault('requires', {}) + for i in context['dependencies']: + requires.setdefault(i, {}) + version['data'] = data = impl.meta('data') + for key in ('mtime', 'seqno', 'blob'): + if key in data: + del data[key] + versions.append(version) result = {'implementations': versions} if distro: diff --git a/sugar_network/node/routes.py b/sugar_network/node/routes.py index b117e98..bc86799 100644 --- a/sugar_network/node/routes.py +++ b/sugar_network/node/routes.py @@ -373,10 +373,12 @@ class NodeRoutes(model.VolumeRoutes, model.FrontRoutes): result = request.call(method=request.method, path=['implementation', impl['guid'], 'data'], response=response) - props = impl.properties( + response.meta = impl.properties( ['guid', 'context', 'license', 'version', 'stability']) - props['data'] = response.meta - response.meta = props + response.meta['data'] = data = impl.meta('data') + for key in ('mtime', 'seqno', 'blob'): + if key in data: + del data[key] return result diff --git a/tests/units/client/cache.py b/tests/units/client/cache.py index 6a34200..6a73f36 100755 --- a/tests/units/client/cache.py +++ b/tests/units/client/cache.py @@ -108,7 +108,7 @@ class CacheTest(tests.Test): cache = Cache(volume) self.assertEqual([], [i for i in cache]) - def test_ensure(self): + def test_ensure_AfterOpen(self): volume = db.Volume('db', [Context, Implementation]) volume['implementation'].create({'data': {'blob_size': 1}, 'guid': '1', 'context': 'context', 'version': '1', 'license': ['GPL'], 'stability': 'stable'}) @@ -142,6 +142,22 @@ class CacheTest(tests.Test): self.assertRaises(RuntimeError, cache.ensure, 2, 0) + def test_ensure_Live(self): + volume = db.Volume('db', [Context, Implementation]) + + cache = Cache(volume) + # To initiate the cache + cache.ensure(0, 0) + + volume['implementation'].create({'data': {'blob_size': 1}, 'guid': '1', 'context': 'context', 'version': '1', 'license': ['GPL'], 'stability': 'stable'}) + cache.checkin('1', 1) + + cache_limit.value = 10 + self.statvfs.f_bfree = 10 + cache.ensure(1, 0) + assert not volume['implementation'].exists('1') + self.assertRaises(RuntimeError, cache.ensure, 1, 0) + def test_ensure_ConsiderTmpSize(self): volume = db.Volume('db', [Context, Implementation]) volume['implementation'].create({'data': {'blob_size': 1}, 'guid': '1', 'context': 'context', 'version': '1', 'license': ['GPL'], 'stability': 'stable'}) @@ -189,55 +205,24 @@ class CacheTest(tests.Test): cache.recycle() def test_checkin(self): - local_volume = self.start_online_client() - conn = IPCConnection() - self.statvfs.f_blocks = 0 + volume = db.Volume('db', [Context, Implementation]) + cache = Cache(volume) - impl1 = conn.upload(['implementation'], StringIO(self.zips(['TestActivity/activity/activity.info', [ - '[Activity]', - 'name = TestActivity', - 'bundle_id = context1', - 'exec = true', - 'icon = icon', - 'activity_version = 1', - 'license = Public Domain', - 'stability = stable', - ]])), cmd='submit', initial=True) - impl2 = conn.upload(['implementation'], StringIO(self.zips(['TestActivity/activity/activity.info', [ - '[Activity]', - 'name = TestActivity', - 'bundle_id = context2', - 'exec = true', - 'icon = icon', - 'activity_version = 1', - 'license = Public Domain', - 'stability = stable', - ]])), cmd='submit', initial=True) - impl3 = conn.upload(['implementation'], StringIO(self.zips(['TestActivity/activity/activity.info', [ - '[Activity]', - 'name = TestActivity', - 'bundle_id = context3', - 'exec = true', - 'icon = icon', - 'activity_version = 1', - 'license = Public Domain', - 'stability = stable', - ]])), cmd='submit', initial=True) + volume['implementation'].create({'guid': '1', 'context': 'context', 'version': '1', 'license': ['GPL'], 'stability': 'stable'}) + volume['implementation'].create({'guid': '2', 'context': 'context', 'version': '1', 'license': ['GPL'], 'stability': 'stable'}) + volume['implementation'].create({'guid': '3', 'context': 'context', 'version': '1', 'license': ['GPL'], 'stability': 'stable'}) - self.assertEqual('exit', [i for i in conn.get(['context', 'context1'], cmd='launch')][-1]['event']) - self.assertEqual([impl1], [i for i in self.client_routes._cache]) - assert local_volume['implementation'].exists(impl1) + cache.checkin('1', 1) + self.assertEqual(['1'], [i for i in cache]) + self.assertEqual(1, cache.du) - self.assertEqual('exit', [i for i in conn.get(['context', 'context2'], cmd='launch')][-1]['event']) - self.assertEqual([impl2, impl1], [i for i in self.client_routes._cache]) - assert local_volume['implementation'].exists(impl1) - assert local_volume['implementation'].exists(impl2) + cache.checkin('2', 2) + self.assertEqual(['2', '1'], [i for i in cache]) + self.assertEqual(3, cache.du) - self.assertEqual('exit', [i for i in conn.get(['context', 'context3'], cmd='launch')][-1]['event']) - self.assertEqual([impl3, impl2, impl1], [i for i in self.client_routes._cache]) - assert local_volume['implementation'].exists(impl1) - assert local_volume['implementation'].exists(impl2) - assert local_volume['implementation'].exists(impl3) + cache.checkin('3', 3) + self.assertEqual(['3', '2', '1'], [i for i in cache]) + self.assertEqual(6, cache.du) def test_checkout(self): local_volume = self.start_online_client() @@ -285,6 +270,48 @@ class CacheTest(tests.Test): assert local_volume['implementation'].exists(impl1) assert local_volume['implementation'].exists(impl2) + def test_Acquiring(self): + volume = db.Volume('db', [Context, Implementation]) + cache = Cache(volume) + + volume['implementation'].create({'guid': '1', 'context': 'context', 'version': '1', 'license': ['GPL'], 'stability': 'stable'}) + volume['implementation'].create({'guid': '2', 'context': 'context', 'version': '1', 'license': ['GPL'], 'stability': 'stable'}) + volume['implementation'].create({'guid': '3', 'context': 'context', 'version': '1', 'license': ['GPL'], 'stability': 'stable'}) + + cache.checkin('1', 1) + self.assertEqual(['1'], [i for i in cache]) + self.assertEqual(1, cache.du) + + cache.acquire('1', 2) + self.assertEqual([], [i for i in cache]) + self.assertEqual(0, cache.du) + cache.acquire('1', 3) + self.assertEqual([], [i for i in cache]) + self.assertEqual(0, cache.du) + cache.acquire('2', 1) + self.assertEqual([], [i for i in cache]) + self.assertEqual(0, cache.du) + cache.acquire('2', 2) + self.assertEqual([], [i for i in cache]) + self.assertEqual(0, cache.du) + cache.acquire('2', 3) + self.assertEqual([], [i for i in cache]) + self.assertEqual(0, cache.du) + + cache.release('1', '2') + self.assertEqual([], [i for i in cache]) + self.assertEqual(0, cache.du) + cache.release('1', '2') + self.assertEqual(['1'], [i for i in cache]) + self.assertEqual(2, cache.du) + cache.release('2') + self.assertEqual(['2', '1'], [i for i in cache]) + self.assertEqual(3, cache.du) + + cache.release('1', '2') + self.assertEqual(['2', '1'], [i for i in cache]) + self.assertEqual(3, cache.du) + if __name__ == '__main__': tests.main() diff --git a/tests/units/client/implementations.py b/tests/units/client/implementations.py index 1374e7c..e63d132 100755 --- a/tests/units/client/implementations.py +++ b/tests/units/client/implementations.py @@ -14,8 +14,8 @@ from os.path import exists, dirname from __init__ import tests -from sugar_network.client import journal, implementations -from sugar_network.toolkit import coroutine, enforce, lsb_release +from sugar_network.client import journal, implementations, cache_limit +from sugar_network.toolkit import coroutine, lsb_release from sugar_network.node import obs from sugar_network.model.user import User from sugar_network.model.context import Context @@ -135,7 +135,7 @@ class Implementations(tests.Test): self.start_online_client() conn = IPCConnection() - impl = conn.upload(['implementation'], StringIO(self.zips(['TestActivity/activity/activity.info', [ + activity_info = '\n'.join([ '[Activity]', 'name = TestActivity', 'bundle_id = bundle_id', @@ -144,15 +144,22 @@ class Implementations(tests.Test): 'activity_version = 1', 'license = Public Domain', 'stability = stable', - ]])), cmd='submit', initial=True) + ]) + blob = self.zips(['TestActivity/activity/activity.info', activity_info]) + impl = conn.upload(['implementation'], StringIO(blob), cmd='submit', initial=True) solution = ['http://127.0.0.1:8888', ['stable'], [{ 'license': ['Public Domain'], 'stability': 'stable', 'version': '1', - 'command': ['true'], 'context': 'bundle_id', 'path': tests.tmpdir + '/client/implementation/%s/%s/data.blob' % (impl[:2], impl), 'guid': impl, + 'data': { + 'unpack_size': len(activity_info), + 'blob_size': len(blob), + 'mime_type': 'application/vnd.olpc-sugar', + 'spec': {'*-*': {'commands': {'activity': {'exec': 'true'}}, 'requires': {}}}, + }, }]] cached_path = 'solutions/bu/bundle_id' @@ -179,10 +186,12 @@ class Implementations(tests.Test): 'license': ['Public Domain'], 'stability': 'stable', 'version': '1', - 'command': ['true'], 'context': 'bundle_id', 'path': tests.tmpdir, 'guid': 'impl', + 'data': { + 'spec': {'*-*': {'commands': {'activity': {'exec': 'true'}}, 'requires': {}}}, + }, }]]) cached_path = 'solutions/bu/bundle_id' self.touch([cached_path, solution]) @@ -242,10 +251,12 @@ class Implementations(tests.Test): 'license': ['Public Domain'], 'stability': 'stable', 'version': '1', - 'command': ['true'], 'context': 'bundle_id', 'path': tests.tmpdir, 'guid': 'impl', + 'data': { + 'spec': {'*-*': {'commands': {'activity': {'exec': 'true'}}, 'requires': {}}}, + }, }]]) self.touch(['solutions/bu/bundle_id', solution]) @@ -392,6 +403,93 @@ class Implementations(tests.Test): self.assertEqual({'en-us': ''}, doc.meta('notes')['value']) self.assertEqual([], doc.meta('tags')['value']) + def test_LaunchAcquiring(self): + volume = self.start_online_client() + conn = IPCConnection() + + app = conn.upload(['implementation'], StringIO(self.zips( + ['TestActivity/activity/activity.info', [ + '[Activity]', + 'name = TestActivity', + 'bundle_id = bundle_id', + 'exec = activity', + 'icon = icon', + 'activity_version = 1', + 'license = Public Domain', + ]], + ['TestActivity/bin/activity', [ + '#!/bin/sh', + 'sleep 1', + ]], + )), cmd='submit', initial=True) + + conn.post(['context'], { + 'guid': 'document', + 'type': 'content', + 'title': 'title', + 'summary': 'summary', + 'description': 'description', + }) + doc = conn.post(['implementation'], { + 'context': 'document', + 'license': 'GPLv3+', + 'version': '1', + 'stability': 'stable', + }) + self.node_volume['implementation'].update(doc, {'data': { + 'mime_type': 'application/octet-stream', + 'blob': StringIO('content'), + }}) + + launch = conn.get(['context', 'document'], cmd='launch', context='bundle_id') + self.assertEqual('launch', next(launch)['event']) + self.assertEqual('exec', next(launch)['event']) + + class statvfs(object): + f_blocks = 100 + f_bfree = 10 + f_frsize = 1 + self.override(os, 'statvfs', lambda *args: statvfs()) + cache_limit.value = 10 + + self.assertRaises(RuntimeError, self.client_routes._cache.ensure, 1, 0) + assert volume['implementation'].exists(app) + assert volume['implementation'].exists(doc) + self.assertEqual([], [i for i in self.client_routes._cache]) + + self.assertEqual('exit', next(launch)['event']) + self.assertEqual([app, doc], [i for i in self.client_routes._cache]) + + def test_NoAcquiringForClones(self): + volume = self.start_online_client() + conn = IPCConnection() + + app = conn.upload(['implementation'], StringIO(self.zips( + ['TestActivity/activity/activity.info', [ + '[Activity]', + 'name = TestActivity', + 'bundle_id = bundle_id', + 'exec = activity', + 'icon = icon', + 'activity_version = 1', + 'license = Public Domain', + ]], + ['TestActivity/bin/activity', [ + '#!/bin/sh', + 'sleep 1', + ]], + )), cmd='submit', initial=True) + + conn.put(['context', 'bundle_id'], True, cmd='clone') + self.assertEqual([], [i for i in self.client_routes._cache]) + + launch = conn.get(['context', 'bundle_id'], cmd='launch') + self.assertEqual('launch', next(launch)['event']) + self.assertEqual('exec', next(launch)['event']) + self.assertEqual([], [i for i in self.client_routes._cache]) + self.assertEqual('exit', next(launch)['event']) + self.assertEqual([], [i for i in self.client_routes._cache]) + if __name__ == '__main__': tests.main() diff --git a/tests/units/client/offline_routes.py b/tests/units/client/offline_routes.py index 2187c1f..5725cf2 100755 --- a/tests/units/client/offline_routes.py +++ b/tests/units/client/offline_routes.py @@ -101,23 +101,26 @@ class OfflineRoutes(tests.Test): 'implementations': [ { 'version': '1', - 'arch': '*-*', 'stability': 'stable', 'guid': impl1, 'license': ['GPLv3+'], + 'data': {'spec': {'*-*': {}}}, }, { 'version': '2', - 'arch': '*-*', 'stability': 'stable', 'guid': impl2, - 'requires': { - 'dep1': {}, - 'dep2': {'restrictions': [['1', '2']]}, - 'dep3': {'restrictions': [[None, '2']]}, - 'dep4': {'restrictions': [['3', None]]}, - }, 'license': ['GPLv3+'], + 'data': { + 'spec': {'*-*': { + 'requires': { + 'dep1': {}, + 'dep2': {'restrictions': [['1', '2']]}, + 'dep3': {'restrictions': [[None, '2']]}, + 'dep4': {'restrictions': [['3', None]]}, + }, + }}, + }, }, ], }, @@ -270,7 +273,7 @@ class OfflineRoutes(tests.Test): local = self.start_online_client() ipc = IPCConnection() - blob = self.zips(['TestActivity/activity/activity.info', [ + activity_info = '\n'.join([ '[Activity]', 'name = TestActivity', 'bundle_id = bundle_id', @@ -278,7 +281,8 @@ class OfflineRoutes(tests.Test): 'icon = icon', 'activity_version = 1', 'license=Public Domain', - ]]) + ]) + blob = self.zips(['TestActivity/activity/activity.info', activity_info]) impl = ipc.upload(['implementation'], StringIO(blob), cmd='submit', initial=True) ipc.put(['context', 'bundle_id'], True, cmd='clone') @@ -288,8 +292,13 @@ class OfflineRoutes(tests.Test): 'license': ['Public Domain'], 'stability': 'stable', 'version': '1', - 'command': ['true'], 'path': tests.tmpdir + '/client/implementation/%s/%s/data.blob' % (impl[:2], impl), + 'data': { + 'unpack_size': len(activity_info), + 'blob_size': len(blob), + 'mime_type': 'application/vnd.olpc-sugar', + 'spec': {'*-*': {'commands': {'activity': {'exec': 'true'}}, 'requires': {}}}, + }, }] assert local['implementation'].exists(impl) self.assertEqual( @@ -422,7 +431,9 @@ Can't find all required implementations: 'license': ['GPLv3+'], 'stability': 'stable', 'version': '1', - 'command': ['true'], + 'data': { + 'spec': {'*-*': {'commands': {'activity': {'exec': 'true'}}, 'requires': {'dep': {}}}}, + }, }, { 'guid': 'dep', 'context': 'dep', @@ -474,7 +485,7 @@ Can't find all required implementations: assert not exists(guid_path) def test_SubmitReport(self): - ipc = self.home_volume = self.start_offline_client([User, Report]) + ipc = self.home_volume = self.start_offline_client() self.touch( ['file1', 'content1'], diff --git a/tests/units/client/online_routes.py b/tests/units/client/online_routes.py index 786b07f..d326223 100755 --- a/tests/units/client/online_routes.py +++ b/tests/units/client/online_routes.py @@ -174,29 +174,38 @@ class OnlineRoutes(tests.Test): 'dep4': {'restrictions': [['3', None]]}, }, }}, + 'blob_size': 1, + 'unpack_size': 2, + 'mime_type': 'foo', }}) self.assertEqual({ 'implementations': [ { 'version': '1', - 'arch': '*-*', 'stability': 'stable', 'guid': impl1, 'license': ['GPLv3+'], + 'data': {'spec': {'*-*': {}}}, }, { 'version': '2', - 'arch': '*-*', 'stability': 'stable', 'guid': impl2, - 'requires': { - 'dep1': {}, - 'dep2': {'restrictions': [['1', '2']]}, - 'dep3': {'restrictions': [[None, '2']]}, - 'dep4': {'restrictions': [['3', None]]}, - }, 'license': ['GPLv3+'], + 'data': { + 'spec': {'*-*': { + 'requires': { + 'dep1': {}, + 'dep2': {'restrictions': [['1', '2']]}, + 'dep3': {'restrictions': [[None, '2']]}, + 'dep4': {'restrictions': [['3', None]]}, + }, + }}, + 'blob_size': 1, + 'unpack_size': 2, + 'mime_type': 'foo', + }, }, ], }, @@ -404,6 +413,7 @@ Can't find all required implementations: 'notes': '', }) self.node_volume['implementation'].update(impl, {'data': { + 'blob_size': 1, 'spec': { '*-*': { 'commands': { @@ -423,13 +433,24 @@ Can't find all required implementations: tests.tmpdir + '/.sugar/default/logs/sugar-network-client.log', ], 'solution': [{ - 'command': ['echo'], - 'context': context, 'guid': impl, + 'context': context, 'license': ['GPLv3+'], 'stability': 'stable', 'version': '1', 'path': tests.tmpdir + '/client/implementation/%s/%s/data.blob' % (impl[:2], impl), + 'data': { + 'spec': { + '*-*': { + 'commands': { + 'activity': { + 'exec': 'echo', + }, + }, + }, + }, + 'blob_size': 1, + }, }], }, ], @@ -463,6 +484,17 @@ Can't find all required implementations: blob = 'content' self.node_volume['implementation'].update(impl, {'data': {'blob': StringIO(blob), 'foo': 'bar'}}) clone_path = 'client/context/%s/%s/.clone' % (context[:2], context) + solution = [{ + 'guid': impl, + 'context': context, + 'license': ['GPLv3+'], + 'version': '1', + 'stability': 'stable', + 'data': { + 'foo': 'bar', + 'blob_size': len(blob), + }, + }] self.assertEqual([ {'event': 'ready'}, @@ -504,6 +536,7 @@ Can't find all required implementations: }, local['implementation'].get(impl).properties(['context', 'license', 'version', 'stability'])) blob_path = 'client/implementation/%s/%s/data.blob' % (impl[:2], impl) + solution[0]['path'] = tests.tmpdir + '/' + blob_path self.assertEqual({ 'seqno': 5, 'blob_size': len(blob), @@ -514,7 +547,9 @@ Can't find all required implementations: local['implementation'].get(impl).meta('data')) self.assertEqual('content', file(blob_path).read()) assert exists(clone_path + '/data.blob') - assert not exists('solutions/%s/%s' % (context[:2], context)) + self.assertEqual( + [client.api_url.value, ['stable'], solution], + json.load(file('solutions/%s/%s' % (context[:2], context)))) self.assertEqual([ ], @@ -549,7 +584,9 @@ Can't find all required implementations: local['implementation'].get(impl).meta('data')) self.assertEqual('content', file(blob_path).read()) assert not lexists(clone_path) - assert not exists('solutions/%s/%s' % (context[:2], context)) + self.assertEqual( + [client.api_url.value, ['stable'], solution], + json.load(file('solutions/%s/%s' % (context[:2], context)))) self.assertEqual([ {'event': 'ready'}, @@ -566,7 +603,9 @@ Can't find all required implementations: sorted([{'guid': context, 'layer': ['clone']}]), sorted(ipc.get(['context'], reply='layer')['result'])) assert exists(clone_path + '/data.blob') - assert not exists('solutions/%s/%s' % (context[:2], context)) + self.assertEqual( + [client.api_url.value, ['stable'], solution], + json.load(file('solutions/%s/%s' % (context[:2], context)))) def test_clone_Activity(self): local = self.start_online_client([User, Context, Implementation]) @@ -598,7 +637,12 @@ Can't find all required implementations: 'license': ['Public Domain'], 'stability': 'stable', 'version': '1', - 'command': ['true'], + 'data': { + 'unpack_size': len(activity_info), + 'blob_size': len(blob), + 'mime_type': 'application/vnd.olpc-sugar', + 'spec': {'*-*': {'commands': {'activity': {'exec': 'true'}}, 'requires': {}}}, + }, }] downloaded_solution = copy.deepcopy(solution) downloaded_solution[0]['path'] = blob_path @@ -804,8 +848,6 @@ Can't find all required implementations: 'data': { 'blob_size': len(blob), 'mime_type': 'application/vnd.olpc-sugar', - 'mtime': int(os.stat(blob_path[:-5]).st_mtime), - 'seqno': 3, 'spec': {'*-*': {'commands': {'activity': {'exec': 'true'}}, 'requires': {}}}, 'unpack_size': len(activity_info), }, @@ -839,7 +881,7 @@ Can't find all required implementations: local = self.start_online_client([User, Context, Implementation]) ipc = IPCConnection() - blob = self.zips(['TestActivity/activity/activity.info', [ + activity_info = '\n'.join([ '[Activity]', 'name = TestActivity', 'bundle_id = bundle_id', @@ -847,7 +889,8 @@ Can't find all required implementations: 'icon = icon', 'activity_version = 1', 'license=Public Domain', - ]]) + ]) + blob = self.zips(['TestActivity/activity/activity.info', activity_info]) impl = ipc.upload(['implementation'], StringIO(blob), cmd='submit', initial=True) coroutine.sleep(.1) @@ -857,7 +900,12 @@ Can't find all required implementations: 'license': ['Public Domain'], 'stability': 'stable', 'version': '1', - 'command': ['true'], + 'data': { + 'blob_size': len(blob), + 'unpack_size': len(activity_info), + 'mime_type': 'application/vnd.olpc-sugar', + 'spec': {'*-*': {'commands': {'activity': {'exec': 'true'}}, 'requires': {}}}, + }, }] downloaded_solution = copy.deepcopy(solution) downloaded_solution[0]['path'] = tests.tmpdir + '/client/implementation/%s/%s/data.blob' % (impl[:2], impl) @@ -892,8 +940,13 @@ Can't find all required implementations: 'license': ['Public Domain'], 'stability': 'stable', 'version': '2', - 'command': ['true'], 'path': tests.tmpdir + '/client/implementation/%s/%s/data.blob' % (impl[:2], impl), + 'data': { + 'blob_size': len(blob), + 'unpack_size': len(activity_info), + 'mime_type': 'application/vnd.olpc-sugar', + 'spec': {'*-*': {'commands': {'activity': {'exec': 'true'}}, 'requires': {}}}, + }, }] log_path = tests.tmpdir + '/.sugar/default/logs/bundle_id_1.log' self.assertEqual([ @@ -984,8 +1037,13 @@ Can't find all required implementations: 'license': ['Public Domain'], 'stability': 'stable', 'version': '1', - 'command': ['false'], 'path': tests.tmpdir + '/client/implementation/%s/%s/data.blob' % (impl[:2], impl), + 'data': { + 'blob_size': len(blob), + 'unpack_size': len(activity_info), + 'mime_type': 'application/vnd.olpc-sugar', + 'spec': {'*-*': {'commands': {'activity': {'exec': 'false'}}, 'requires': {}}}, + }, }] self.assertEqual([ {'event': 'launch', 'foo': 'bar', 'activity_id': 'activity_id'}, @@ -1130,9 +1188,9 @@ Can't find all required implementations: 'implementations': [{ 'stability': 'stable', 'guid': impl, - 'arch': '*-*', 'version': '1', 'license': ['GPLv3+'], + 'data': {'spec': {'*-*': {}}}, }], }, ipc.get(['context', context], cmd='feed')) @@ -1144,9 +1202,9 @@ Can't find all required implementations: 'implementations': [{ 'stability': 'stable', 'guid': impl, - 'arch': '*-*', 'version': '1', 'license': ['GPLv3+'], + 'data': {'spec': {'*-*': {}}}, }], }, ipc.get(['context', context], cmd='feed', layer='public')) @@ -1185,9 +1243,9 @@ Can't find all required implementations: 'implementations': [{ 'stability': 'stable', 'guid': impl, - 'arch': '*-*', 'version': '1', 'license': ['GPLv3+'], + 'data': {'spec': {'*-*': {}}}, }], }, ipc.get(['context', context], cmd='feed', layer='public')) @@ -1201,7 +1259,7 @@ Can't find all required implementations: def blob(self, value): raise http.Redirect(URL) - self.start_online_client([User, Document]) + self.start_online_client([User, Context, Implementation, Document]) ipc = IPCConnection() guid = ipc.post(['document'], {}) @@ -1245,7 +1303,7 @@ Can't find all required implementations: yield '"local"' self.override(routes, '_LocalRoutes', LocalRoutes) - home_volume = self.start_client([User]) + home_volume = self.start_client() ipc = IPCConnection() self.assertEqual('local', ipc.get(cmd='sleep')) @@ -1313,8 +1371,8 @@ Can't find all required implementations: def test_ReconnectOnServerFall(self): routes._RECONNECT_TIMEOUT = 1 - node_pid = self.fork_master([User]) - self.start_client([User]) + node_pid = self.fork_master() + self.start_client() ipc = IPCConnection() self.wait_for_events(ipc, event='inline', state='online').wait() @@ -1324,7 +1382,7 @@ Can't find all required implementations: coroutine.spawn(shutdown) self.wait_for_events(ipc, event='inline', state='offline').wait() - self.fork_master([User]) + self.fork_master() self.wait_for_events(ipc, event='inline', state='online').wait() def test_SilentReconnectOnGatewayErrors(self): @@ -1349,8 +1407,8 @@ Can't find all required implementations: else: raise http.GatewayTimeout() - node_pid = self.start_master([User], Routes) - self.start_client([User]) + node_pid = self.start_master(None, Routes) + self.start_client() ipc = IPCConnection() self.wait_for_events(ipc, event='inline', state='online').wait() @@ -1386,7 +1444,7 @@ Can't find all required implementations: assert not cp.inline() def test_SubmitReport(self): - self.home_volume = self.start_online_client([User, Report]) + self.home_volume = self.start_online_client([User, Context, Implementation, Report]) ipc = IPCConnection() self.touch( diff --git a/tests/units/client/routes.py b/tests/units/client/routes.py index af2b74c..7514c96 100755 --- a/tests/units/client/routes.py +++ b/tests/units/client/routes.py @@ -397,10 +397,6 @@ class RoutesTest(tests.Test): assert exists('client/implementation/%s/%s' % (guid[:2], guid)) - - - - def call(routes, request): router = Router(routes) return router.call(request, Response()) diff --git a/tests/units/client/solver.py b/tests/units/client/solver.py index b3d9666..88ee931 100755 --- a/tests/units/client/solver.py +++ b/tests/units/client/solver.py @@ -106,7 +106,10 @@ class SolverTest(tests.Test): {'version': '1', 'guid': 'dep1', 'context': 'dep1', 'stability': 'packaged', 'license': None}, {'version': '1', 'guid': 'dep2', 'context': 'dep2', 'stability': 'packaged', 'license': None}, {'version': '1', 'guid': 'dep3', 'context': 'dep3', 'stability': 'packaged', 'license': None}, - {'version': '1', 'command': ['echo'], 'context': context, 'guid': impl, 'stability': 'stable', 'license': ['GPLv3+']}, + {'version': '1', 'context': context, 'guid': impl, 'stability': 'stable', 'license': ['GPLv3+'], + 'data': {'spec': {'*-*': {'commands': {'activity': {'exec': 'echo'}}, 'requires': + {'dep2': {'restrictions': [['1', '2']]}, 'dep3': {}}}}}, + 'requires': {'dep1': {}, 'dep2': {}}}, ]), sorted(solver.solve(self.client_routes.fallback, context, ['stable']))) @@ -155,7 +158,8 @@ class SolverTest(tests.Test): }, }}) self.assertEqual([ - {'version': '1', 'command': ['echo'], 'context': context, 'guid': impl, 'stability': 'stable', 'license': ['GPLv3+']}, + {'version': '1', 'context': context, 'guid': impl, 'stability': 'stable', 'license': ['GPLv3+'], + 'data': {'spec': {'*-*': {'commands': {'activity': {'exec': 'echo'}}, 'requires': {'sugar': {}}}}}}, {'version': '0.94', 'context': 'sugar', 'guid': 'sugar-0.94', 'stability': 'packaged', 'license': None}, ], solver.solve(self.client_routes.fallback, context, ['stable'])) @@ -175,7 +179,9 @@ class SolverTest(tests.Test): }, }}) self.assertEqual([ - {'version': '1', 'command': ['echo'], 'context': context, 'guid': impl, 'stability': 'stable', 'license': ['GPLv3+']}, + {'version': '1', 'context': context, 'guid': impl, 'stability': 'stable', 'license': ['GPLv3+'], + 'data': {'spec': {'*-*': {'commands': {'activity': {'exec': 'echo'}}, 'requires': + {'sugar': {'restrictions': [['0.80', '0.87']]}}}}}}, {'version': '0.86', 'context': 'sugar', 'guid': 'sugar-0.86', 'stability': 'packaged', 'license': None}, ], solver.solve(self.client_routes.fallback, context, ['stable'])) @@ -225,7 +231,8 @@ class SolverTest(tests.Test): }, }}) self.assertEqual([ - {'version': '1', 'command': ['echo'], 'context': context, 'guid': impl, 'stability': 'stable', 'license': ['GPLv3+']}, + {'version': '1', 'context': context, 'guid': impl, 'stability': 'stable', 'license': ['GPLv3+'], + 'data': {'spec': {'*-*': {'commands': {'activity': {'exec': 'echo'}}, 'requires': {'sugar': {}}}}}}, {'version': '0.94', 'context': 'sugar', 'guid': 'sugar-0.94', 'stability': 'packaged', 'license': None}, ], solver.solve(self.client_routes.fallback, context, ['stable'])) diff --git a/tests/units/node/node.py b/tests/units/node/node.py index 1925a90..c39d185 100755 --- a/tests/units/node/node.py +++ b/tests/units/node/node.py @@ -650,8 +650,6 @@ class NodeTest(tests.Test): 'version': '3', 'license': ['GPLv3+'], 'data': { - 'seqno': 8, - 'mtime': int(os.stat('master/implementation/%s/%s/data.blob' % (impl3[:2], impl3)).st_mtime), 'blob_size': len(blob3), 'spec': { '*-*': { -- cgit v0.9.1