Initial BLOBs cache implementation

7 files changed, 249 insertions, 155 deletions

diff --git a/TODO b/TODO
index 59c9da0..9a149e4 100644
--- a/TODO
+++ b/TODO
@@ -1,3 +1,5 @@
 - import <bundle_id> data dir into data/
 - reuse the same data dir for all SN names for the same context
 - treat "/" in user datain requests
+- GC blobs cache
+- add suffixes for cached BLOB files to bot break logic
diff --git a/sugar_network/_zerosugar/solution.py b/sugar_network/_zerosugar/solution.py
index a81a5cd..ba1973f 100644
--- a/sugar_network/_zerosugar/solution.py
+++ b/sugar_network/_zerosugar/solution.py
@@ -13,11 +13,8 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import os
-import zipfile
-from os.path import join, exists, dirname
+from os.path import join
 
-from sugar_network import sugar, util
 from sugar_network.resources import Implementation
 
 
@@ -128,22 +125,11 @@ class _Selection(object):
         return self._value.dependencies
 
     def download(self):
-        path = sugar.profile_path('implementations', self.id)
-        if not exists(path):
-            tmp_path = util.TempFilePath(dir=dirname(path))
-            with file(tmp_path, 'wb') as f:
-                impl = Implementation(self.id)
-                for chunk in impl.blobs['bundle'].iter_content():
-                    f.write(chunk)
-                if not f.tell():
-                    return
-            bundle = zipfile.ZipFile(tmp_path)
-            bundle.extractall(path)
-
-        top_files = os.listdir(path)
-        if len(top_files) == 1:
-            path = join(path, top_files[0])
-        self._value.impl.local_path = path
+        if not self.download_sources:
+            return
+        impl = Implementation(self.id)
+        self._value.impl.local_path = join(impl.blobs['bundle'].path,
+                self.download_sources[0].extract)
 
     def __getattr__(self, name):
         return getattr(self._value.impl, name)
diff --git a/sugar_network/blobs.py b/sugar_network/blobs.py
new file mode 100644
index 0000000..aed137b
--- /dev/null
+++ b/sugar_network/blobs.py
@@ -0,0 +1,116 @@
+# Copyright (C) 2012, Aleksey Lim
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import json
+import shutil
+import logging
+import tempfile
+from os.path import isdir, exists
+
+from sweets_recipe import Bundle
+
+from sugar_network import sugar
+from sugar_network.request import request, raw_request
+
+
+_CHUNK_SIZE = 1024 * 10
+
+_logger = logging.getLogger('client')
+
+
+class Blob(object):
+
+    def __init__(self, path):
+        self._path = path
+
+    @property
+    def content(self):
+        """Return entire BLOB value as a string."""
+        path, mime_path = self._get()
+        with file(mime_path) as f:
+            mime_type = f.read().strip()
+        with file(path) as f:
+            if mime_type == 'application/json':
+                return json.load(f)
+            else:
+                return f.read()
+
+    @property
+    def path(self):
+        """Return file-system path to file that contain BLOB value."""
+        path, __ = self._get()
+        return path
+
+    def iter_content(self):
+        """Return BLOB value by poritons.
+
+        :returns:
+            generator that returns BLOB value by chunks
+
+        """
+        path, __ = self._get()
+        with file(path) as f:
+            while True:
+                chunk = f.read(_CHUNK_SIZE)
+                if not chunk:
+                    break
+                yield chunk
+
+    def _set_url(self, url):
+        request('PUT', self._path, params={'url': url})
+
+    #: Set BLOB value by url
+    url = property(None, _set_url)
+
+    def _get(self):
+        path = sugar.profile_path('cache', 'blobs', *self._path)
+        mime_path = path + '.mime'
+
+        if exists(path) and exists(mime_path):
+            return path, mime_path
+
+        if isdir(path):
+            shutil.rmtree(path)
+
+        response = raw_request('GET', self._path, allow_redirects=True)
+
+        with file(mime_path, 'w') as f:
+            f.write(response.headers.get('Content-Type') or \
+                    'application/octet-stream')
+
+        def download(f):
+            _logger.debug('Download "%s" BLOB to %s', self._path, path)
+
+            length = int(response.headers.get('Content-Length', _CHUNK_SIZE))
+            chunk_size = min(length, _CHUNK_SIZE)
+
+            for chunk in response.iter_content(chunk_size=chunk_size):
+                f.write(chunk)
+
+        if self._path[0] == 'implementation' and self._path[-1] == 'bundle':
+            tmp_file = tempfile.NamedTemporaryFile(delete=False)
+            try:
+                download(tmp_file)
+                tmp_file.close()
+                with Bundle(tmp_file.name, 'application/zip') as bundle:
+                    bundle.extractall(path)
+            finally:
+                os.unlink(tmp_file.name)
+        else:
+            with file(path, 'w') as f:
+                download(f)
+
+        return path, mime_path
diff --git a/sugar_network/client.py b/sugar_network/client.py
index f65f79b..296f57f 100644
--- a/sugar_network/client.py
+++ b/sugar_network/client.py
@@ -13,36 +13,24 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import json
 import logging
-import hashlib
 import collections
 from gettext import gettext as _
 
-import requests
-from M2Crypto import DSA
-
-from sugar_network import env, sugar
+from sugar_network import sugar
+from sugar_network.blobs import Blob
+from sugar_network.request import request
 from sugar_network.util import enforce
 
 
 _PAGE_SIZE = 16
 _PAGE_NUMBER = 5
-_CHUNK_SIZE = 1024 * 10
 
 _logger = logging.getLogger('client')
-_headers = {}
 
 
 def delete(resource, guid):
-    _request('DELETE', [resource, guid])
-
-
-class ServerError(Exception):
-
-    def __init__(self, request_, error):
-        self.request = request_
-        Exception.__init__(self, error)
+    request('DELETE', [resource, guid])
 
 
 class Query(object):
@@ -195,7 +183,7 @@ class Query(object):
         if self._reply_properties:
             params['reply'] = ','.join(self._reply_properties)
 
-        reply = _request('GET', self._path, params=params)
+        reply = request('GET', self._path, params=params)
         self._total = reply['total']
 
         result = [None] * len(reply['result'])
@@ -243,7 +231,7 @@ class Object(dict):
         result = self.get(prop)
         if result is None:
             if self._path and not self._got:
-                reply = _request('GET', self._path)
+                reply = request('GET', self._path)
                 reply.update(self)
                 self.update(reply)
                 self._got = True
@@ -266,7 +254,7 @@ class Object(dict):
         for i in self._dirty:
             data[i] = self[i]
         if 'guid' in self:
-            _request('PUT', self._path, data=data,
+            request('PUT', self._path, data=data,
                     headers={'Content-Type': 'application/json'})
         else:
             if 'author' in data:
@@ -275,7 +263,7 @@ class Object(dict):
             else:
                 data['author'] = [sugar.guid()]
                 dict.__setitem__(self, 'author', [sugar.guid()])
-            reply = _request('POST', [self._resource], data=data,
+            reply = request('POST', [self._resource], data=data,
                     headers={'Content-Type': 'application/json'})
             self.update(reply)
             self._path = [self._resource, self['guid']]
@@ -284,45 +272,7 @@ class Object(dict):
     def call(self, command, method='GET', **kwargs):
         enforce(self._path is not None, _('Object needs to be posted first'))
         kwargs['cmd'] = command
-        return _request(method, self._path, params=kwargs)
-
-
-class Blob(object):
-
-    def __init__(self, path):
-        self._path = path
-
-    @property
-    def content(self):
-        """Return entire BLOB value as a string."""
-        response = _request('GET', self._path, allow_redirects=True)
-        if hasattr(response, 'content'):
-            return response.content
-        else:
-            return response
-
-    @property
-    def path(self):
-        """Return file-system path to file that contain BLOB value."""
-        return '/home/me/Activities/cartoon-builder.activity/' \
-                'activity/activity-cartoonbuilder.svg'
-
-    def iter_content(self):
-        """Return BLOB value by poritons.
-
-        :returns:
-            generator that returns BLOB value by chunks
-
-        """
-        response = _request('GET', self._path, allow_redirects=True)
-        length = int(response.headers.get('Content-Length', _CHUNK_SIZE))
-        return response.iter_content(chunk_size=min(length, _CHUNK_SIZE))
-
-    def _set_url(self, url):
-        _request('PUT', self._path, params={'url': url})
-
-    #: Set BLOB value by url
-    url = property(None, _set_url)
+        return request(method, self._path, params=kwargs)
 
 
 class _Blobs(object):
@@ -344,72 +294,5 @@ class _Blobs(object):
         else:
             files = None
             headers = {'Content-Type': 'application/octet-stream'}
-        _request('PUT', self._path + [prop], headers=headers,
+        request('PUT', self._path + [prop], headers=headers,
                 data=data, files=files)
-
-
-def _request(method, path, data=None, headers=None, **kwargs):
-    path = '/'.join([i.strip('/') for i in [env.api_url.value] + path])
-
-    if not _headers:
-        uid = sugar.guid()
-        _headers['sugar_user'] = uid
-        _headers['sugar_user_signature'] = _sign(uid)
-    if headers:
-        headers.update(_headers)
-    else:
-        headers = _headers
-
-    if data is not None and headers.get('Content-Type') == 'application/json':
-        data = json.dumps(data)
-
-    verify = True
-    if env.no_check_certificate.value:
-        verify = False
-    elif env.certfile.value:
-        verify = env.certfile.value
-
-    while True:
-        try:
-            response = requests.request(method, path, data=data, verify=verify,
-                    headers=headers, config={'keep_alive': True}, **kwargs)
-        except requests.exceptions.SSLError:
-            _logger.warning(_('Pass --no-check-certificate ' \
-                    'to avoid SSL checks'))
-            raise
-
-        if response.status_code != 200:
-            if response.status_code == 401:
-                _register()
-                continue
-            content = response.content
-            try:
-                error = json.loads(content)
-                raise ServerError(error['request'], error['error'])
-            except ValueError:
-                _logger.debug('Got %s HTTP error for "%s" request:\n%s',
-                        response.status_code, path, content)
-                response.raise_for_status()
-
-        if response.headers.get('Content-Type') == 'application/json':
-            return json.loads(response.content)
-        else:
-            return response
-
-
-def _register():
-    _request('POST', ['user'],
-            headers={'Content-Type': 'application/json'},
-            data={
-                'nickname': sugar.nickname() or '',
-                'color': sugar.color() or '#000000,#000000',
-                'machine_sn': sugar.machine_sn() or '',
-                'machine_uuid': sugar.machine_uuid() or '',
-                'pubkey': sugar.pubkey(),
-                },
-            )
-
-
-def _sign(data):
-    key = DSA.load_key(sugar.profile_path('owner.key'))
-    return key.sign_asn1(hashlib.sha1(data).digest()).encode('hex')
diff --git a/sugar_network/request.py b/sugar_network/request.py
new file mode 100644
index 0000000..0ca92d1
--- /dev/null
+++ b/sugar_network/request.py
@@ -0,0 +1,107 @@
+# Copyright (C) 2012, Aleksey Lim
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+import logging
+import hashlib
+from gettext import gettext as _
+
+import requests
+from M2Crypto import DSA
+
+from sugar_network import env, sugar
+
+
+_logger = logging.getLogger('client')
+_headers = {}
+
+
+class ServerError(Exception):
+
+    def __init__(self, request_, error):
+        self.request = request_
+        Exception.__init__(self, error)
+
+
+def request(method, path, data=None, headers=None, **kwargs):
+    response = raw_request(method, path, data, headers, **kwargs)
+    if response.headers.get('Content-Type') == 'application/json':
+        return json.loads(response.content)
+    else:
+        return response
+
+
+def raw_request(method, path, data=None, headers=None, **kwargs):
+    path = '/'.join([i.strip('/') for i in [env.api_url.value] + path])
+
+    if not _headers:
+        uid = sugar.guid()
+        _headers['sugar_user'] = uid
+        _headers['sugar_user_signature'] = _sign(uid)
+    if headers:
+        headers.update(_headers)
+    else:
+        headers = _headers
+
+    if data is not None and headers.get('Content-Type') == 'application/json':
+        data = json.dumps(data)
+
+    verify = True
+    if env.no_check_certificate.value:
+        verify = False
+    elif env.certfile.value:
+        verify = env.certfile.value
+
+    while True:
+        try:
+            response = requests.request(method, path, data=data, verify=verify,
+                    headers=headers, config={'keep_alive': True}, **kwargs)
+        except requests.exceptions.SSLError:
+            _logger.warning(_('Pass --no-check-certificate ' \
+                    'to avoid SSL checks'))
+            raise
+
+        if response.status_code != 200:
+            if response.status_code == 401:
+                _register()
+                continue
+            content = response.content
+            try:
+                error = json.loads(content)
+                raise ServerError(error['request'], error['error'])
+            except ValueError:
+                _logger.debug('Got %s HTTP error for "%s" request:\n%s',
+                        response.status_code, path, content)
+                response.raise_for_status()
+
+        return response
+
+
+def _register():
+    raw_request('POST', ['user'],
+            headers={'Content-Type': 'application/json'},
+            data={
+                'nickname': sugar.nickname() or '',
+                'color': sugar.color() or '#000000,#000000',
+                'machine_sn': sugar.machine_sn() or '',
+                'machine_uuid': sugar.machine_uuid() or '',
+                'pubkey': sugar.pubkey(),
+                },
+            )
+
+
+def _sign(data):
+    key = DSA.load_key(sugar.profile_path('owner.key'))
+    return key.sign_asn1(hashlib.sha1(data).digest()).encode('hex')
diff --git a/tests/__init__.py b/tests/__init__.py
index 85bd6d5..6501b54 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -125,7 +125,7 @@ class Test(unittest.TestCase):
         server.index_flush_timeout.value = 0
         server.index_flush_threshold.value = 1
 
-        node = ad.Master(server.resources())
+        node = ad.Master(server.resources)
         httpd = WSGIServer(('localhost', port), rd.Router(node))
 
         try:
diff --git a/tests/units/client.py b/tests/units/client.py
index e7c8f81..4318faf 100755
--- a/tests/units/client.py
+++ b/tests/units/client.py
@@ -16,14 +16,14 @@ class ClientTest(tests.Test):
                 'response': [],
                 }
 
-        def request(method, path, data=None, params=None):
-            self.stat['requests'].append((method, path, params))
+        def request(method, path, data=None, params=None, headers=None):
+            self.stat['requests'].append((method, '/' + '/'.join(path), params))
 
             if params is None:
                 if self.stat['response']:
                     return self.stat['response'].pop(0)
                 else:
-                    return {'guid': path.split('/')[-1],
+                    return {'guid': path[-1],
                             'prop': 'value',
                             }
 
@@ -83,10 +83,10 @@ class ClientTest(tests.Test):
                 self.stat['requests'][10:])
 
     def test_Object_Gets(self):
-        obj = client.Object('resource', {'guid': 1})
+        obj = client.Object('resource', {'guid': '1'})
         self.assertEqual([], self.stat['requests'])
 
-        self.assertEqual(1, obj['guid'])
+        self.assertEqual('1', obj['guid'])
         self.assertEqual([], self.stat['requests'])
 
         self.assertEqual('value', obj['prop'])
@@ -97,7 +97,7 @@ class ClientTest(tests.Test):
         self.assertRaises(KeyError, lambda: obj['foo'])
         self.assertEqual([('GET', '/resource/1', None)], self.stat['requests'])
 
-        obj = client.Object('resource', {'guid': 2})
+        obj = client.Object('resource', {'guid': '2'})
         self.assertRaises(KeyError, lambda: obj['foo'])
         self.assertEqual([('GET', '/resource/2', None)], self.stat['requests'][1:])
         self.assertRaises(KeyError, lambda: obj['foo'])
@@ -154,7 +154,7 @@ class ClientTest(tests.Test):
     def test_Object_DoNotOverrideSetsAfterPost(self):
         obj = client.Object('resource')
         obj['foo'] = 'bar'
-        self.stat['response'].append({'guid': 1})
+        self.stat['response'].append({'guid': '1'})
         obj.post()
 
         self.assertEqual(