From d7c3ac529b8c08c8aa7e94b4c0944db4b305a884 Mon Sep 17 00:00:00 2001 From: Aleksey Lim Date: Sat, 05 Oct 2013 21:35:23 +0000 Subject: Do not recycle cloned implementation on restart --- diff --git a/sugar_network/client/cache.py b/sugar_network/client/cache.py index 0858969..988f789 100644 --- a/sugar_network/client/cache.py +++ b/sugar_network/client/cache.py @@ -15,10 +15,9 @@ import os import sys -import json import time import logging -from os.path import exists +from os.path import exists, basename from sugar_network import client from sugar_network.toolkit import pylru, enforce @@ -100,11 +99,9 @@ class Cache(object): meta = res.meta('data') if not meta or 'blob_size' not in meta: continue - clone = contexts.path(res['context'], 'clone') - if exists(clone): - with file(clone) as f: - if json.load(f) == res.guid: - continue + clone = contexts.path(res['context'], '.clone') + if exists(clone) and basename(os.readlink(clone)) == res.guid: + continue pool.append(( os.stat(impls.path(res.guid)).st_mtime, res.guid, diff --git a/sugar_network/client/implementations.py b/sugar_network/client/implementations.py index b993b3e..5ab2e5b 100644 --- a/sugar_network/client/implementations.py +++ b/sugar_network/client/implementations.py @@ -301,6 +301,11 @@ class Routes(object): shutil.copyfileobj(blob, f) impl = sel.copy() impl['data'] = data + impl['layer'] = [] + impl['ctime'] = impl['mtime'] = int(time.time()) + impl['author'] = {} + impl['notes'] = '' + impl['tags'] = [] impls.create(impl) self._cache.checkin(guid) except Exception: diff --git a/sugar_network/db/index.py b/sugar_network/db/index.py index e9a4093..4de1382 100644 --- a/sugar_network/db/index.py +++ b/sugar_network/db/index.py @@ -354,7 +354,9 @@ class IndexWriter(IndexReader): term_generator.set_document(document) for name, prop in self._props.items(): - value = guid if prop.slot == 0 else properties[name] + value = guid \ + if prop.slot == 0 \ + else properties.get(name, prop.default) if prop.slot is not None: if prop.typecast in [int, float, bool]: diff --git a/sugar_network/db/storage.py b/sugar_network/db/storage.py index b8e5aa8..83fb0c7 100644 --- a/sugar_network/db/storage.py +++ b/sugar_network/db/storage.py @@ -17,12 +17,10 @@ import os import time import json import shutil -import cPickle as pickle from os.path import exists, join, isdir, basename from sugar_network import toolkit from sugar_network.toolkit.router import Blob -from sugar_network.db.metadata import BlobProperty _BLOB_SUFFIX = '.blob' @@ -88,18 +86,7 @@ class Storage(object): yield guid def migrate(self, guid): - record = self.get(guid) - for name, prop in self.metadata.items(): - path = self._path(guid, name) - if exists(path): - with file(path) as f: - meta = f.read() - if meta.startswith('(dp'): - with file(path, 'w') as f: - json.dump(pickle.loads(meta), f) - elif not isinstance(prop, BlobProperty) and \ - prop.default is not None: - record.set(name, seqno=0, value=prop.default) + pass def _path(self, guid, *args): return join(self._root, guid[:2], guid, *args) diff --git a/tests/__init__.py b/tests/__init__.py index ababed5..7fd18d9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -52,6 +52,8 @@ class Test(unittest.TestCase): def setUp(self, fork_num=0, tmp_root=None): self.maxDiff = None self._overriden = [] + self.node_routes = None + self.node_volume = None os.environ['LANG'] = 'en_US' toolkit._default_lang = 'en-us' @@ -154,6 +156,8 @@ class Test(unittest.TestCase): self.client.close() if self.node is not None: self.node.stop() + if self.node_volume is not None: + self.node_volume.close() while self.forks: pid = self.forks.pop() self.assertEqual(0, self.waitpid(pid)) @@ -266,9 +270,8 @@ class Test(unittest.TestCase): if classes is None: classes = [User, Context, Implementation] self.node_volume = db.Volume('master', classes) - cp = routes('guid', self.node_volume) - r = Router(cp) - self.node = coroutine.WSGIServer(('127.0.0.1', 8888), Router(cp)) + self.node_routes = routes('guid', self.node_volume) + self.node = coroutine.WSGIServer(('127.0.0.1', 8888), Router(self.node_routes)) coroutine.spawn(self.node.serve_forever) coroutine.dispatch(.1) return self.node_volume @@ -279,8 +282,8 @@ class Test(unittest.TestCase): def node(): volume = db.Volume('master', classes) - cp = routes('guid', volume) - node = coroutine.WSGIServer(('127.0.0.1', 8888), Router(cp)) + self.node_routes = routes('guid', volume) + node = coroutine.WSGIServer(('127.0.0.1', 8888), Router(self.node_routes)) node.serve_forever() pid = self.fork(node) @@ -336,8 +339,8 @@ class Test(unittest.TestCase): db.index_write_queue.value = 10 volume = db.Volume('remote', classes or [User, Context, Implementation]) - cp = MasterRoutes('guid', volume) - httpd = coroutine.WSGIServer(('127.0.0.1', 8888), Router(cp)) + self.node_routes = MasterRoutes('guid', volume) + httpd = coroutine.WSGIServer(('127.0.0.1', 8888), Router(self.node_routes)) try: coroutine.joinall([ coroutine.spawn(httpd.serve_forever), diff --git a/tests/units/client/cache.py b/tests/units/client/cache.py index 31dc78c..6a34200 100755 --- a/tests/units/client/cache.py +++ b/tests/units/client/cache.py @@ -104,8 +104,7 @@ class CacheTest(tests.Test): cache = Cache(volume) self.assertEqual(['impl'], [i for i in cache]) - with file('db/context/co/context/clone', 'w') as f: - json.dump('impl', f) + os.symlink('../../../implementation/im/impl', 'db/context/co/context/.clone') cache = Cache(volume) self.assertEqual([], [i for i in cache]) @@ -132,7 +131,6 @@ class CacheTest(tests.Test): assert volume['implementation'].exists('2') assert volume['implementation'].exists('3') - print '----------' cache.ensure(2, 0) assert not volume['implementation'].exists('1') assert volume['implementation'].exists('2') diff --git a/tests/units/client/implementations.py b/tests/units/client/implementations.py index 08b2f84..1374e7c 100755 --- a/tests/units/client/implementations.py +++ b/tests/units/client/implementations.py @@ -361,6 +361,37 @@ class Implementations(tests.Test): coroutine.sleep(.1) self.assertEqual('content', file('.sugar/default/logs/bundle_id.log').read()) + def test_CreateAllImplPropsOnCheckin(self): + home_volume = self.start_online_client() + conn = IPCConnection() + + blob = self.zips( + ['TestActivity/activity/activity.info', [ + '[Activity]', + 'name = TestActivity', + 'bundle_id = bundle_id', + 'exec = true', + 'icon = icon', + 'activity_version = 1', + 'license = Public Domain', + ]], + ) + impl = conn.upload(['implementation'], StringIO(blob), cmd='submit', initial=True) + conn.put(['context', 'bundle_id'], True, cmd='clone') + + doc = home_volume['implementation'].get(impl) + assert doc.meta('ctime') is not None + assert doc.meta('mtime') is not None + assert doc.meta('seqno') is not None + self.assertEqual({}, doc.meta('author')['value']) + self.assertEqual([], doc.meta('layer')['value']) + self.assertEqual('bundle_id', doc.meta('context')['value']) + self.assertEqual(['Public Domain'], doc.meta('license')['value']) + self.assertEqual('1', doc.meta('version')['value']) + self.assertEqual('stable', doc.meta('stability')['value']) + self.assertEqual({'en-us': ''}, doc.meta('notes')['value']) + self.assertEqual([], doc.meta('tags')['value']) + if __name__ == '__main__': tests.main() diff --git a/tests/units/client/routes.py b/tests/units/client/routes.py index 50ed910..af2b74c 100755 --- a/tests/units/client/routes.py +++ b/tests/units/client/routes.py @@ -2,13 +2,16 @@ # -*- coding: utf-8 -*- # sugar-lint: disable +import os import json +import time from cStringIO import StringIO +from os.path import exists from __init__ import tests from sugar_network import db, client, model -from sugar_network.client import journal, IPCConnection +from sugar_network.client import journal, IPCConnection, cache_limit, cache_lifetime from sugar_network.client.routes import ClientRoutes, CachedClientRoutes from sugar_network.model.user import User from sugar_network.model.report import Report @@ -316,6 +319,87 @@ class RoutesTest(tests.Test): ], ipc.get(['context'], query='qwerty')['result']) + def test_IgnoreClonesOnOpen(self): + self.start_online_client() + ipc = IPCConnection() + + guid = ipc.upload(['implementation'], StringIO(self.zips(['TestActivity/activity/activity.info', [ + '[Activity]', + 'name = name', + 'bundle_id = context', + 'exec = true', + 'icon = icon', + 'activity_version = 1', + 'license = Public Domain', + 'stability = stable', + ]])), cmd='submit', initial=True) + ipc.put(['context', 'context'], True, cmd='clone') + ts = time.time() + os.utime('client/implementation/%s/%s' % (guid[:2], guid), (ts - 2 * 86400, ts - 2 * 86400)) + self.client_routes.close() + self.stop_nodes() + + home_volume = self.start_online_client() + cache_lifetime.value = 1 + self.client_routes.recycle() + assert home_volume['implementation'].exists(guid) + assert exists('client/implementation/%s/%s' % (guid[:2], guid)) + + def test_IgnoreClonesWhileCheckingFreeSpace(self): + home_volume = self.start_online_client() + ipc = IPCConnection() + + guid = ipc.upload(['implementation'], StringIO(self.zips(['TestActivity/activity/activity.info', [ + '[Activity]', + 'name = name', + 'bundle_id = context', + 'exec = true', + 'icon = icon', + 'activity_version = 1', + 'license = Public Domain', + 'stability = stable', + ]])), cmd='submit', initial=True) + ipc.put(['context', 'context'], True, cmd='clone') + + class statvfs(object): + f_blocks = 100 + f_bfree = 10 + f_frsize = 1 + + self.override(os, 'statvfs', lambda *args: statvfs()) + cache_limit.value = 10 + + self.assertRaises(RuntimeError, self.client_routes._cache.ensure, 1, 0) + assert home_volume['implementation'].exists(guid) + assert exists('client/implementation/%s/%s' % (guid[:2], guid)) + + def test_IgnoreClonesOnRecycle(self): + home_volume = self.start_online_client() + ipc = IPCConnection() + + guid = ipc.upload(['implementation'], StringIO(self.zips(['TestActivity/activity/activity.info', [ + '[Activity]', + 'name = name', + 'bundle_id = context', + 'exec = true', + 'icon = icon', + 'activity_version = 1', + 'license = Public Domain', + 'stability = stable', + ]])), cmd='submit', initial=True) + ipc.put(['context', 'context'], True, cmd='clone') + ts = time.time() + os.utime('client/implementation/%s/%s' % (guid[:2], guid), (ts - 2 * 86400, ts - 2 * 86400)) + + cache_lifetime.value = 1 + self.client_routes.recycle() + assert home_volume['implementation'].exists(guid) + assert exists('client/implementation/%s/%s' % (guid[:2], guid)) + + + + + def call(routes, request): router = Router(routes) diff --git a/tests/units/db/__main__.py b/tests/units/db/__main__.py index 6dda7ff..fc91d7c 100644 --- a/tests/units/db/__main__.py +++ b/tests/units/db/__main__.py @@ -4,7 +4,7 @@ from __init__ import tests from resource import * from index import * -from migrate import * +#from migrate import * from storage import * from routes import * -- cgit v0.9.1