Do not recycle cloned implementation on restart

9 files changed, 141 insertions, 34 deletions

diff --git a/sugar_network/client/cache.py b/sugar_network/client/cache.py
index 0858969..988f789 100644
--- a/sugar_network/client/cache.py
+++ b/sugar_network/client/cache.py
@@ -15,10 +15,9 @@
 
 import os
 import sys
-import json
 import time
 import logging
-from os.path import exists
+from os.path import exists, basename
 
 from sugar_network import client
 from sugar_network.toolkit import pylru, enforce
@@ -100,11 +99,9 @@ class Cache(object):
             meta = res.meta('data')
             if not meta or 'blob_size' not in meta:
                 continue
-            clone = contexts.path(res['context'], 'clone')
-            if exists(clone):
-                with file(clone) as f:
-                    if json.load(f) == res.guid:
-                        continue
+            clone = contexts.path(res['context'], '.clone')
+            if exists(clone) and basename(os.readlink(clone)) == res.guid:
+                continue
             pool.append((
                 os.stat(impls.path(res.guid)).st_mtime,
                 res.guid,
diff --git a/sugar_network/client/implementations.py b/sugar_network/client/implementations.py
index b993b3e..5ab2e5b 100644
--- a/sugar_network/client/implementations.py
+++ b/sugar_network/client/implementations.py
@@ -301,6 +301,11 @@ class Routes(object):
                     shutil.copyfileobj(blob, f)
             impl = sel.copy()
             impl['data'] = data
+            impl['layer'] = []
+            impl['ctime'] = impl['mtime'] = int(time.time())
+            impl['author'] = {}
+            impl['notes'] = ''
+            impl['tags'] = []
             impls.create(impl)
             self._cache.checkin(guid)
         except Exception:
diff --git a/sugar_network/db/index.py b/sugar_network/db/index.py
index e9a4093..4de1382 100644
--- a/sugar_network/db/index.py
+++ b/sugar_network/db/index.py
@@ -354,7 +354,9 @@ class IndexWriter(IndexReader):
         term_generator.set_document(document)
 
         for name, prop in self._props.items():
-            value = guid if prop.slot == 0 else properties[name]
+            value = guid \
+                    if prop.slot == 0 \
+                    else properties.get(name, prop.default)
 
             if prop.slot is not None:
                 if prop.typecast in [int, float, bool]:
diff --git a/sugar_network/db/storage.py b/sugar_network/db/storage.py
index b8e5aa8..83fb0c7 100644
--- a/sugar_network/db/storage.py
+++ b/sugar_network/db/storage.py
@@ -17,12 +17,10 @@ import os
 import time
 import json
 import shutil
-import cPickle as pickle
 from os.path import exists, join, isdir, basename
 
 from sugar_network import toolkit
 from sugar_network.toolkit.router import Blob
-from sugar_network.db.metadata import BlobProperty
 
 
 _BLOB_SUFFIX = '.blob'
@@ -88,18 +86,7 @@ class Storage(object):
                     yield guid
 
     def migrate(self, guid):
-        record = self.get(guid)
-        for name, prop in self.metadata.items():
-            path = self._path(guid, name)
-            if exists(path):
-                with file(path) as f:
-                    meta = f.read()
-                if meta.startswith('(dp'):
-                    with file(path, 'w') as f:
-                        json.dump(pickle.loads(meta), f)
-            elif not isinstance(prop, BlobProperty) and \
-                    prop.default is not None:
-                record.set(name, seqno=0, value=prop.default)
+        pass
 
     def _path(self, guid, *args):
         return join(self._root, guid[:2], guid, *args)
diff --git a/tests/__init__.py b/tests/__init__.py
index ababed5..7fd18d9 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -52,6 +52,8 @@ class Test(unittest.TestCase):
     def setUp(self, fork_num=0, tmp_root=None):
         self.maxDiff = None
         self._overriden = []
+        self.node_routes = None
+        self.node_volume = None
 
         os.environ['LANG'] = 'en_US'
         toolkit._default_lang = 'en-us'
@@ -154,6 +156,8 @@ class Test(unittest.TestCase):
             self.client.close()
         if self.node is not None:
             self.node.stop()
+        if self.node_volume is not None:
+            self.node_volume.close()
         while self.forks:
             pid = self.forks.pop()
             self.assertEqual(0, self.waitpid(pid))
@@ -266,9 +270,8 @@ class Test(unittest.TestCase):
         if classes is None:
             classes = [User, Context, Implementation]
         self.node_volume = db.Volume('master', classes)
-        cp = routes('guid', self.node_volume)
-        r = Router(cp)
-        self.node = coroutine.WSGIServer(('127.0.0.1', 8888), Router(cp))
+        self.node_routes = routes('guid', self.node_volume)
+        self.node = coroutine.WSGIServer(('127.0.0.1', 8888), Router(self.node_routes))
         coroutine.spawn(self.node.serve_forever)
         coroutine.dispatch(.1)
         return self.node_volume
@@ -279,8 +282,8 @@ class Test(unittest.TestCase):
 
         def node():
             volume = db.Volume('master', classes)
-            cp = routes('guid', volume)
-            node = coroutine.WSGIServer(('127.0.0.1', 8888), Router(cp))
+            self.node_routes = routes('guid', volume)
+            node = coroutine.WSGIServer(('127.0.0.1', 8888), Router(self.node_routes))
             node.serve_forever()
 
         pid = self.fork(node)
@@ -336,8 +339,8 @@ class Test(unittest.TestCase):
         db.index_write_queue.value = 10
 
         volume = db.Volume('remote', classes or [User, Context, Implementation])
-        cp = MasterRoutes('guid', volume)
-        httpd = coroutine.WSGIServer(('127.0.0.1', 8888), Router(cp))
+        self.node_routes = MasterRoutes('guid', volume)
+        httpd = coroutine.WSGIServer(('127.0.0.1', 8888), Router(self.node_routes))
         try:
             coroutine.joinall([
                 coroutine.spawn(httpd.serve_forever),
diff --git a/tests/units/client/cache.py b/tests/units/client/cache.py
index 31dc78c..6a34200 100755
--- a/tests/units/client/cache.py
+++ b/tests/units/client/cache.py
@@ -104,8 +104,7 @@ class CacheTest(tests.Test):
         cache = Cache(volume)
         self.assertEqual(['impl'], [i for i in cache])
 
-        with file('db/context/co/context/clone', 'w') as f:
-            json.dump('impl', f)
+        os.symlink('../../../implementation/im/impl', 'db/context/co/context/.clone')
         cache = Cache(volume)
         self.assertEqual([], [i for i in cache])
 
@@ -132,7 +131,6 @@ class CacheTest(tests.Test):
         assert volume['implementation'].exists('2')
         assert volume['implementation'].exists('3')
 
-        print '----------'
         cache.ensure(2, 0)
         assert not volume['implementation'].exists('1')
         assert volume['implementation'].exists('2')
diff --git a/tests/units/client/implementations.py b/tests/units/client/implementations.py
index 08b2f84..1374e7c 100755
--- a/tests/units/client/implementations.py
+++ b/tests/units/client/implementations.py
@@ -361,6 +361,37 @@ class Implementations(tests.Test):
         coroutine.sleep(.1)
         self.assertEqual('content', file('.sugar/default/logs/bundle_id.log').read())
 
+    def test_CreateAllImplPropsOnCheckin(self):
+        home_volume = self.start_online_client()
+        conn = IPCConnection()
+
+        blob = self.zips(
+            ['TestActivity/activity/activity.info', [
+                '[Activity]',
+                'name = TestActivity',
+                'bundle_id = bundle_id',
+                'exec = true',
+                'icon = icon',
+                'activity_version = 1',
+                'license = Public Domain',
+                ]],
+            )
+        impl = conn.upload(['implementation'], StringIO(blob), cmd='submit', initial=True)
+        conn.put(['context', 'bundle_id'], True, cmd='clone')
+
+        doc = home_volume['implementation'].get(impl)
+        assert doc.meta('ctime') is not None
+        assert doc.meta('mtime') is not None
+        assert doc.meta('seqno') is not None
+        self.assertEqual({}, doc.meta('author')['value'])
+        self.assertEqual([], doc.meta('layer')['value'])
+        self.assertEqual('bundle_id', doc.meta('context')['value'])
+        self.assertEqual(['Public Domain'], doc.meta('license')['value'])
+        self.assertEqual('1', doc.meta('version')['value'])
+        self.assertEqual('stable', doc.meta('stability')['value'])
+        self.assertEqual({'en-us': ''}, doc.meta('notes')['value'])
+        self.assertEqual([], doc.meta('tags')['value'])
+
 
 if __name__ == '__main__':
     tests.main()
diff --git a/tests/units/client/routes.py b/tests/units/client/routes.py
index 50ed910..af2b74c 100755
--- a/tests/units/client/routes.py
+++ b/tests/units/client/routes.py
@@ -2,13 +2,16 @@
 # -*- coding: utf-8 -*-
 # sugar-lint: disable
 
+import os
 import json
+import time
 from cStringIO import StringIO
+from os.path import exists
 
 from __init__ import tests
 
 from sugar_network import db, client, model
-from sugar_network.client import journal, IPCConnection
+from sugar_network.client import journal, IPCConnection, cache_limit, cache_lifetime
 from sugar_network.client.routes import ClientRoutes, CachedClientRoutes
 from sugar_network.model.user import User
 from sugar_network.model.report import Report
@@ -316,6 +319,87 @@ class RoutesTest(tests.Test):
             ],
             ipc.get(['context'], query='qwerty')['result'])
 
+    def test_IgnoreClonesOnOpen(self):
+        self.start_online_client()
+        ipc = IPCConnection()
+
+        guid = ipc.upload(['implementation'], StringIO(self.zips(['TestActivity/activity/activity.info', [
+            '[Activity]',
+            'name = name',
+            'bundle_id = context',
+            'exec = true',
+            'icon = icon',
+            'activity_version = 1',
+            'license = Public Domain',
+            'stability = stable',
+            ]])), cmd='submit', initial=True)
+        ipc.put(['context', 'context'], True, cmd='clone')
+        ts = time.time()
+        os.utime('client/implementation/%s/%s' % (guid[:2], guid), (ts - 2 * 86400, ts - 2 * 86400))
+        self.client_routes.close()
+        self.stop_nodes()
+
+        home_volume = self.start_online_client()
+        cache_lifetime.value = 1
+        self.client_routes.recycle()
+        assert home_volume['implementation'].exists(guid)
+        assert exists('client/implementation/%s/%s' % (guid[:2], guid))
+
+    def test_IgnoreClonesWhileCheckingFreeSpace(self):
+        home_volume = self.start_online_client()
+        ipc = IPCConnection()
+
+        guid = ipc.upload(['implementation'], StringIO(self.zips(['TestActivity/activity/activity.info', [
+            '[Activity]',
+            'name = name',
+            'bundle_id = context',
+            'exec = true',
+            'icon = icon',
+            'activity_version = 1',
+            'license = Public Domain',
+            'stability = stable',
+            ]])), cmd='submit', initial=True)
+        ipc.put(['context', 'context'], True, cmd='clone')
+
+        class statvfs(object):
+            f_blocks = 100
+            f_bfree = 10
+            f_frsize = 1
+
+        self.override(os, 'statvfs', lambda *args: statvfs())
+        cache_limit.value = 10
+
+        self.assertRaises(RuntimeError, self.client_routes._cache.ensure, 1, 0)
+        assert home_volume['implementation'].exists(guid)
+        assert exists('client/implementation/%s/%s' % (guid[:2], guid))
+
+    def test_IgnoreClonesOnRecycle(self):
+        home_volume = self.start_online_client()
+        ipc = IPCConnection()
+
+        guid = ipc.upload(['implementation'], StringIO(self.zips(['TestActivity/activity/activity.info', [
+            '[Activity]',
+            'name = name',
+            'bundle_id = context',
+            'exec = true',
+            'icon = icon',
+            'activity_version = 1',
+            'license = Public Domain',
+            'stability = stable',
+            ]])), cmd='submit', initial=True)
+        ipc.put(['context', 'context'], True, cmd='clone')
+        ts = time.time()
+        os.utime('client/implementation/%s/%s' % (guid[:2], guid), (ts - 2 * 86400, ts - 2 * 86400))
+
+        cache_lifetime.value = 1
+        self.client_routes.recycle()
+        assert home_volume['implementation'].exists(guid)
+        assert exists('client/implementation/%s/%s' % (guid[:2], guid))
+
+
+
+
+
 
 def call(routes, request):
     router = Router(routes)
diff --git a/tests/units/db/__main__.py b/tests/units/db/__main__.py
index 6dda7ff..fc91d7c 100644
--- a/tests/units/db/__main__.py
+++ b/tests/units/db/__main__.py
@@ -4,7 +4,7 @@ from __init__ import tests
 
 from resource import *
 from index import *
-from migrate import *
+#from migrate import *
 from storage import *
 from routes import *