From 47f1ea9d473075774f6825c546566b2ac48ad25b Mon Sep 17 00:00:00 2001 From: Aleksey Lim Date: Sun, 05 Jan 2014 09:47:44 +0000 Subject: Return blobs metadata in GET prop requests --- diff --git a/sugar_network/client/cache.py b/sugar_network/client/cache.py index 554dde4..07daa16 100644 --- a/sugar_network/client/cache.py +++ b/sugar_network/client/cache.py @@ -138,7 +138,7 @@ class Cache(object): stat = os.statvfs(client.local_root.value) if stat.f_blocks == 0: - # TODO Sonds like a tmpfs or so + # TODO Sounds like a tmpfs or so return 0 limit = sys.maxint diff --git a/sugar_network/client/implementations.py b/sugar_network/client/implementations.py index f78df33..ca9e89c 100644 --- a/sugar_network/client/implementations.py +++ b/sugar_network/client/implementations.py @@ -24,6 +24,7 @@ import random import shutil import hashlib import logging +from copy import deepcopy from os.path import join, exists, basename, dirname, relpath from sugar_network import client, toolkit @@ -227,7 +228,7 @@ class Routes(object): def cache_impl(sel): guid = sel['guid'] data = sel['data'] - data_path = sel['path'] = impls.path(guid, 'data') + sel['path'] = impls.path(guid, 'data') size = data.get('unpack_size') or data['blob_size'] blob = None @@ -240,33 +241,36 @@ class Routes(object): if blob is None: blob = self._call(method='GET', path=['implementation', guid, 'data']) - try: - if not exists(dirname(data_path)): - os.makedirs(dirname(data_path)) + + blob_dir = dirname(sel['path']) + if not exists(blob_dir): + os.makedirs(blob_dir) + + with toolkit.mkdtemp(dir=blob_dir) as blob_dir: if 'activity' in context['type']: self._cache.ensure(size, data['blob_size']) with toolkit.TemporaryFile() as tmp_file: shutil.copyfileobj(blob, tmp_file) tmp_file.seek(0) with Bundle(tmp_file, 'application/zip') as bundle: - bundle.extractall(data_path, prefix=bundle.rootdir) + bundle.extractall(blob_dir, prefix=bundle.rootdir) for exec_dir in ('bin', 'activity'): - bin_path = join(data_path, exec_dir) + bin_path = join(blob_dir, exec_dir) if not exists(bin_path): continue for filename in os.listdir(bin_path): os.chmod(join(bin_path, filename), 0755) + blob = blob_dir else: self._cache.ensure(size) - with file(data_path, 'wb') as f: + with file(join(blob_dir, 'data'), 'wb') as f: shutil.copyfileobj(blob, f) - impl = sel.copy() + blob = f.name + impl = deepcopy(sel) impl['mtime'] = impl['ctime'] + impl['data']['blob'] = blob impls.create(impl) return cache_call(guid, size) - except Exception: - shutil.rmtree(data_path, ignore_errors=True) - raise result = [] for sel in request.session['solution']: diff --git a/sugar_network/client/journal.py b/sugar_network/client/journal.py index 646bced..ee2a2f3 100644 --- a/sugar_network/client/journal.py +++ b/sugar_network/client/journal.py @@ -90,7 +90,7 @@ class Routes(object): # Do not break SN like API guid = item['guid'] = item.pop('uid') if has_preview: - item['preview'] = _preview_url(guid) + item['preview'] = _preview(guid) return {'result': result, 'total': int(total)} @@ -100,7 +100,7 @@ class Routes(object): return {'guid': guid, 'title': get(guid, 'title'), 'description': get(guid, 'description'), - 'preview': _preview_url(guid), + 'preview': _preview(guid), } @route('GET', ['journal', None, 'preview']) @@ -193,6 +193,7 @@ def _prop_path(guid, prop): return _ds_path(guid, 'metadata', prop) -def _preview_url(guid): - return 'http://127.0.0.1:%s/journal/%s/preview' % \ - (client.ipc_port.value, guid) +def _preview(guid): + return {'url': 'http://127.0.0.1:%s/journal/%s/preview' % + (client.ipc_port.value, guid), + } diff --git a/sugar_network/db/directory.py b/sugar_network/db/directory.py index eb2f0e2..96a2923 100644 --- a/sugar_network/db/directory.py +++ b/sugar_network/db/directory.py @@ -313,7 +313,7 @@ class Directory(object): if op is not None: op(patch) for prop, meta in merge.items(): - is_blob = isinstance(self.metadata[prop], BlobProperty) + is_blob = isinstance(self.metadata.get(prop), BlobProperty) record.set(prop, cleanup_blob=is_blob, **meta) if record.consistent: diff --git a/sugar_network/db/routes.py b/sugar_network/db/routes.py index 77e2c4b..123e001 100644 --- a/sugar_network/db/routes.py +++ b/sugar_network/db/routes.py @@ -267,11 +267,16 @@ class Routes(object): if value is None: value = prop.default elif isinstance(value, Blob): - value = value.get('url') - if value is None: - value = '/'.join(['', metadata.name, doc.guid, name]) - if value.startswith('/'): - value = request.static_prefix + value + for key in ('mtime', 'seqno', 'blob'): + if key in value: + del value[key] + url = value.get('url') + if url is None: + value['url'] = '/'.join([ + request.static_prefix, metadata.name, doc.guid, name, + ]) + elif url.startswith('/'): + value['url'] = request.static_prefix + url result[name] = value return result diff --git a/sugar_network/toolkit/__init__.py b/sugar_network/toolkit/__init__.py index 5988e5d..a32d87f 100644 --- a/sugar_network/toolkit/__init__.py +++ b/sugar_network/toolkit/__init__.py @@ -485,7 +485,7 @@ def unique_filename(root, filename): class mkdtemp(str): def __new__(cls, **kwargs): - if cachedir.value: + if cachedir.value and 'dir' not in kwargs: if not exists(cachedir.value): os.makedirs(cachedir.value) kwargs['dir'] = cachedir.value @@ -496,7 +496,8 @@ class mkdtemp(str): return self def __exit__(self, exc_type, exc_value, traceback): - shutil.rmtree(self) + if exists(self): + shutil.rmtree(self) def svg_to_png(data, w, h): @@ -522,7 +523,7 @@ def svg_to_png(data, w, h): def TemporaryFile(*args, **kwargs): - if cachedir.value: + if cachedir.value and 'dir' not in kwargs: if not exists(cachedir.value): os.makedirs(cachedir.value) kwargs['dir'] = cachedir.value @@ -532,7 +533,7 @@ def TemporaryFile(*args, **kwargs): class NamedTemporaryFile(object): def __init__(self, *args, **kwargs): - if cachedir.value: + if cachedir.value and 'dir' not in kwargs: if not exists(cachedir.value): os.makedirs(cachedir.value) kwargs['dir'] = cachedir.value diff --git a/tests/units/client/journal.py b/tests/units/client/journal.py index fd36a2a..30c67f8 100755 --- a/tests/units/client/journal.py +++ b/tests/units/client/journal.py @@ -107,32 +107,32 @@ class JournalTest(tests.Test): request.path = ['journal'] response = Response() self.assertEqual([ - {'guid': 'guid1', 'title': 'title1', 'description': 'description1', 'preview': url + 'guid1/preview'}, - {'guid': 'guid2', 'title': 'title2', 'description': 'description2', 'preview': url + 'guid2/preview'}, - {'guid': 'guid3', 'title': 'title3', 'description': 'description3', 'preview': url + 'guid3/preview'}, + {'guid': 'guid1', 'title': 'title1', 'description': 'description1', 'preview': {'url': url + 'guid1/preview'}}, + {'guid': 'guid2', 'title': 'title2', 'description': 'description2', 'preview': {'url': url + 'guid2/preview'}}, + {'guid': 'guid3', 'title': 'title3', 'description': 'description3', 'preview': {'url': url + 'guid3/preview'}}, ], ds.journal_find(request, response)['result']) request = Request(offset=1, limit=1, reply=['uid', 'title', 'description', 'preview']) request.path = ['journal'] self.assertEqual([ - {'guid': 'guid2', 'title': 'title2', 'description': 'description2', 'preview': url + 'guid2/preview'}, + {'guid': 'guid2', 'title': 'title2', 'description': 'description2', 'preview': {'url': url + 'guid2/preview'}}, ], ds.journal_find(request, response)['result']) request = Request(query='title3', reply=['uid', 'title', 'description', 'preview']) request.path = ['journal'] self.assertEqual([ - {'guid': 'guid3', 'title': 'title3', 'description': 'description3', 'preview': url + 'guid3/preview'}, + {'guid': 'guid3', 'title': 'title3', 'description': 'description3', 'preview': {'url': url + 'guid3/preview'}}, ], ds.journal_find(request, response)['result']) request = Request(order_by=['+title'], reply=['uid', 'title', 'description', 'preview']) request.path = ['journal'] self.assertEqual([ - {'guid': 'guid3', 'title': 'title3', 'description': 'description3', 'preview': url + 'guid3/preview'}, - {'guid': 'guid2', 'title': 'title2', 'description': 'description2', 'preview': url + 'guid2/preview'}, - {'guid': 'guid1', 'title': 'title1', 'description': 'description1', 'preview': url + 'guid1/preview'}, + {'guid': 'guid3', 'title': 'title3', 'description': 'description3', 'preview': {'url': url + 'guid3/preview'}}, + {'guid': 'guid2', 'title': 'title2', 'description': 'description2', 'preview': {'url': url + 'guid2/preview'}}, + {'guid': 'guid1', 'title': 'title1', 'description': 'description1', 'preview': {'url': url + 'guid1/preview'}}, ], ds.journal_find(request, response)['result']) @@ -146,7 +146,7 @@ class JournalTest(tests.Test): request.path = ['journal', 'guid1'] response = Response() self.assertEqual( - {'guid': 'guid1', 'title': 'title1', 'description': 'description1', 'preview': url + 'guid1/preview'}, + {'guid': 'guid1', 'title': 'title1', 'description': 'description1', 'preview': {'url': url + 'guid1/preview'}}, ds.journal_get(request, response)) def test_GetPropRequest(self): diff --git a/tests/units/client/offline_routes.py b/tests/units/client/offline_routes.py index a163769..866e5d9 100755 --- a/tests/units/client/offline_routes.py +++ b/tests/units/client/offline_routes.py @@ -146,27 +146,48 @@ class OfflineRoutes(tests.Test): 'summary': 'summary', 'description': 'description', }) - ipc.request('PUT', ['context', guid, 'preview'], 'image') + blob = 'preview_blob' + ipc.request('PUT', ['context', guid, 'preview'], blob) self.assertEqual( - 'image', + blob, ipc.request('GET', ['context', guid, 'preview']).content) - self.assertEqual( - {'preview': 'http://127.0.0.1:5555/context/%s/preview' % guid}, - ipc.get(['context', guid], reply=['preview'])) - self.assertEqual( - [{'preview': 'http://127.0.0.1:5555/context/%s/preview' % guid}], - ipc.get(['context'], reply=['preview'])['result']) + self.assertEqual({ + 'preview': { + 'url': 'http://127.0.0.1:5555/context/%s/preview' % guid, + 'blob_size': len(blob), + 'digest': hashlib.sha1(blob).hexdigest(), + 'mime_type': 'image/png', + }, + }, + ipc.get(['context', guid], reply=['preview'])) + self.assertEqual([{ + 'preview': { + 'url': 'http://127.0.0.1:5555/context/%s/preview' % guid, + 'blob_size': len(blob), + 'digest': hashlib.sha1(blob).hexdigest(), + 'mime_type': 'image/png', + }, + }], + ipc.get(['context'], reply=['preview'])['result']) self.assertEqual( file(src_root + '/sugar_network/static/httpdocs/images/package.png').read(), ipc.request('GET', ['context', guid, 'icon']).content) - self.assertEqual( - {'icon': 'http://127.0.0.1:5555/static/images/package.png'}, - ipc.get(['context', guid], reply=['icon'])) - self.assertEqual( - [{'icon': 'http://127.0.0.1:5555/static/images/package.png'}], - ipc.get(['context'], reply=['icon'])['result']) + self.assertEqual({ + 'icon': { + 'url': 'http://127.0.0.1:5555/static/images/package.png', + 'mime_type': 'image/png', + }, + }, + ipc.get(['context', guid], reply=['icon'])) + self.assertEqual([{ + 'icon': { + 'url': 'http://127.0.0.1:5555/static/images/package.png', + 'mime_type': 'image/png', + }, + }], + ipc.get(['context'], reply=['icon'])['result']) def test_favorite(self): ipc = self.start_offline_client() diff --git a/tests/units/client/online_routes.py b/tests/units/client/online_routes.py index ddf8a4c..064a9bb 100755 --- a/tests/units/client/online_routes.py +++ b/tests/units/client/online_routes.py @@ -233,27 +233,48 @@ class OnlineRoutes(tests.Test): 'summary': 'summary', 'description': 'description', }) - ipc.request('PUT', ['context', guid, 'preview'], 'image') + blob = 'preview_blob' + ipc.request('PUT', ['context', guid, 'preview'], blob, headers={'content-type': 'image/png'}) self.assertEqual( - 'image', + blob, ipc.request('GET', ['context', guid, 'preview']).content) - self.assertEqual( - {'preview': 'http://127.0.0.1:8888/context/%s/preview' % guid}, - ipc.get(['context', guid], reply=['preview'])) - self.assertEqual( - [{'preview': 'http://127.0.0.1:8888/context/%s/preview' % guid}], - ipc.get(['context'], reply=['preview'])['result']) + self.assertEqual({ + 'preview': { + 'url': 'http://127.0.0.1:8888/context/%s/preview' % guid, + 'blob_size': len(blob), + 'digest': hashlib.sha1(blob).hexdigest(), + 'mime_type': 'image/png', + }, + }, + ipc.get(['context', guid], reply=['preview'])) + self.assertEqual([{ + 'preview': { + 'url': 'http://127.0.0.1:8888/context/%s/preview' % guid, + 'blob_size': len(blob), + 'digest': hashlib.sha1(blob).hexdigest(), + 'mime_type': 'image/png', + }, + }], + ipc.get(['context'], reply=['preview'])['result']) self.assertEqual( file(src_root + '/sugar_network/static/httpdocs/images/package.png').read(), ipc.request('GET', ['context', guid, 'icon']).content) - self.assertEqual( - {'icon': 'http://127.0.0.1:8888/static/images/package.png'}, - ipc.get(['context', guid], reply=['icon'])) - self.assertEqual( - [{'icon': 'http://127.0.0.1:8888/static/images/package.png'}], - ipc.get(['context'], reply=['icon'])['result']) + self.assertEqual({ + 'icon': { + 'url': 'http://127.0.0.1:8888/static/images/package.png', + 'mime_type': 'image/png', + }, + }, + ipc.get(['context', guid], reply=['icon'])) + self.assertEqual([{ + 'icon': { + 'url': 'http://127.0.0.1:8888/static/images/package.png', + 'mime_type': 'image/png', + }, + }], + ipc.get(['context'], reply=['icon'])['result']) def test_favorite(self): local = self.start_online_client() diff --git a/tests/units/client/server_routes.py b/tests/units/client/server_routes.py index 47282ce..6bbb955 100755 --- a/tests/units/client/server_routes.py +++ b/tests/units/client/server_routes.py @@ -3,6 +3,7 @@ import os import shutil +import hashlib from os.path import exists from __init__ import tests, src_root @@ -129,27 +130,48 @@ class ServerRoutesTest(tests.Test): 'summary': 'summary', 'description': 'description', }) - ipc.request('PUT', ['context', guid, 'preview'], 'image') + blob = 'preview_blob' + ipc.request('PUT', ['context', guid, 'preview'], blob) self.assertEqual( - 'image', + blob, ipc.request('GET', ['context', guid, 'preview']).content) - self.assertEqual( - {'preview': 'http://127.0.0.1:5555/context/%s/preview' % guid}, - ipc.get(['context', guid], reply=['preview'])) - self.assertEqual( - [{'preview': 'http://127.0.0.1:5555/context/%s/preview' % guid}], - ipc.get(['context'], reply=['preview'])['result']) + self.assertEqual({ + 'preview': { + 'url': 'http://127.0.0.1:5555/context/%s/preview' % guid, + 'blob_size': len(blob), + 'digest': hashlib.sha1(blob).hexdigest(), + 'mime_type': 'image/png', + }, + }, + ipc.get(['context', guid], reply=['preview'])) + self.assertEqual([{ + 'preview': { + 'url': 'http://127.0.0.1:5555/context/%s/preview' % guid, + 'blob_size': len(blob), + 'digest': hashlib.sha1(blob).hexdigest(), + 'mime_type': 'image/png', + }, + }], + ipc.get(['context'], reply=['preview'])['result']) self.assertEqual( file(src_root + '/sugar_network/static/httpdocs/images/package.png').read(), ipc.request('GET', ['context', guid, 'icon']).content) - self.assertEqual( - {'icon': 'http://127.0.0.1:5555/static/images/package.png'}, - ipc.get(['context', guid], reply=['icon'])) - self.assertEqual( - [{'icon': 'http://127.0.0.1:5555/static/images/package.png'}], - ipc.get(['context'], reply=['icon'])['result']) + self.assertEqual({ + 'icon': { + 'url': 'http://127.0.0.1:5555/static/images/package.png', + 'mime_type': 'image/png', + }, + }, + ipc.get(['context', guid], reply=['icon'])) + self.assertEqual([{ + 'icon': { + 'url': 'http://127.0.0.1:5555/static/images/package.png', + 'mime_type': 'image/png', + }, + }], + ipc.get(['context'], reply=['icon'])['result']) def test_PopulateNode(self): os.makedirs('disk/sugar-network') diff --git a/tests/units/db/routes.py b/tests/units/db/routes.py index 95aeb4e..51cd892 100755 --- a/tests/units/db/routes.py +++ b/tests/units/db/routes.py @@ -254,28 +254,40 @@ class RoutesTest(tests.Test): self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) guid = self.call('POST', path=['testdocument'], content={}) - self.call('PUT', path=['testdocument', guid, 'blob'], content='blob') + blob = 'blob' + self.call('PUT', path=['testdocument', guid, 'blob'], content=blob) blob_path = tests.tmpdir + '/testdocument/%s/%s/blob' % (guid[:2], guid) blob_meta = { 'seqno': 2, 'blob': blob_path + '.blob', - 'blob_size': 4, - 'digest': hashlib.sha1('blob').hexdigest(), + 'blob_size': len(blob), + 'digest': hashlib.sha1(blob).hexdigest(), 'mime_type': 'application/octet-stream', 'mtime': int(os.stat(blob_path).st_mtime), } self.assertEqual('blob', file(self.call('GET', path=['testdocument', guid, 'blob'])['blob']).read()) - self.assertEqual( - {'guid': guid, 'blob': 'http://localhost/testdocument/%s/blob' % guid}, - self.call('GET', path=['testdocument', guid], reply=['guid', 'blob'], host='localhost')) - - self.assertEqual([ - {'guid': guid, 'blob': 'http://localhost/testdocument/%s/blob' % guid}, - ], - self.call('GET', path=['testdocument'], reply=['guid', 'blob'], host='localhost')['result']) + self.assertEqual({ + 'blob': { + 'url': 'http://localhost/testdocument/%s/blob' % guid, + 'blob_size': len(blob), + 'digest': hashlib.sha1(blob).hexdigest(), + 'mime_type': u'application/octet-stream', + }, + }, + self.call('GET', path=['testdocument', guid], reply=['blob'], host='localhost')) + + self.assertEqual([{ + 'blob': { + 'url': 'http://localhost/testdocument/%s/blob' % guid, + 'blob_size': len(blob), + 'digest': hashlib.sha1(blob).hexdigest(), + 'mime_type': u'application/octet-stream', + }, + }], + self.call('GET', path=['testdocument'], reply=['blob'], host='localhost')['result']) def test_GetBLOBsByUrls(self): @@ -286,37 +298,51 @@ class RoutesTest(tests.Test): return value self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) - guid = self.call('POST', path=['testdocument'], content={}) + guid1 = self.call('POST', path=['testdocument'], content={}) - self.assertRaises(http.NotFound, self.call, 'GET', path=['testdocument', guid, 'blob']) + self.assertRaises(http.NotFound, self.call, 'GET', path=['testdocument', guid1, 'blob']) self.assertEqual( - {'blob': 'http://127.0.0.1/testdocument/%s/blob' % guid}, - self.call('GET', path=['testdocument', guid], reply=['blob'], host='127.0.0.1')) - self.assertEqual([ - {'blob': 'http://127.0.0.1/testdocument/%s/blob' % guid}, - ], - self.call('GET', path=['testdocument'], reply=['blob'], host='127.0.0.1')['result']) + {'blob': {'url': 'http://127.0.0.1/testdocument/%s/blob' % guid1}}, + self.call('GET', path=['testdocument', guid1], reply=['blob'], host='127.0.0.1')) - self.call('PUT', path=['testdocument', guid, 'blob'], content='file') - self.assertEqual('file', file(self.call('GET', path=['testdocument', guid, 'blob'])['blob']).read()) - self.assertEqual( - {'blob': 'http://127.0.0.1/testdocument/%s/blob' % guid}, - self.call('GET', path=['testdocument', guid], reply=['blob'], host='127.0.0.1')) - self.assertEqual([ - {'blob': 'http://127.0.0.1/testdocument/%s/blob' % guid}, - ], - self.call('GET', path=['testdocument'], reply=['blob'], host='127.0.0.1')['result']) + blob = 'file' + guid2 = self.call('POST', path=['testdocument'], content={'blob': blob}) + self.assertEqual('file', file(self.call('GET', path=['testdocument', guid2, 'blob'])['blob']).read()) + self.assertEqual({ + 'blob': { + 'url': 'http://127.0.0.1/testdocument/%s/blob' % guid2, + 'blob_size': len(blob), + 'digest': hashlib.sha1(blob).hexdigest(), + 'mime_type': u'application/octet-stream', + }, + }, + self.call('GET', path=['testdocument', guid2], reply=['blob'], host='127.0.0.1')) + + guid3 = self.call('POST', path=['testdocument'], content={'blob': {'url': 'http://foo'}}, content_type='application/json') + self.assertEqual('http://foo', self.call('GET', path=['testdocument', guid3, 'blob'])['url']) + self.assertEqual({ + 'blob': { + 'url': 'http://foo', + }, + }, + self.call('GET', path=['testdocument', guid3], reply=['blob'], host='127.0.0.1')) - self.call('PUT', path=['testdocument', guid, 'blob'], content={'url': 'http://foo'}, - content_type='application/json') - self.assertEqual('http://foo', self.call('GET', path=['testdocument', guid, 'blob'])['url']) self.assertEqual( - {'blob': 'http://foo'}, - self.call('GET', path=['testdocument', guid], reply=['blob'], host='127.0.0.1')) - self.assertEqual([ - {'blob': 'http://foo'}, - ], - self.call('GET', path=['testdocument'], reply=['blob'], host='127.0.0.1')['result']) + sorted([ + {'blob': { + 'url': 'http://127.0.0.1/testdocument/%s/blob' % guid1, + }}, + { 'blob': { + 'url': 'http://127.0.0.1/testdocument/%s/blob' % guid2, + 'blob_size': len(blob), + 'digest': hashlib.sha1(blob).hexdigest(), + 'mime_type': u'application/octet-stream', + }}, + { 'blob': { + 'url': 'http://foo', + }}, + ]), + sorted(self.call('GET', path=['testdocument'], reply=['blob'], host='127.0.0.1')['result'])) def test_CommandsGetAbsentBlobs(self): @@ -336,7 +362,7 @@ class RoutesTest(tests.Test): self.assertEqual('value', self.call('GET', path=['testdocument', guid, 'prop'])) self.assertRaises(http.NotFound, self.call, 'GET', path=['testdocument', guid, 'blob']) self.assertEqual( - {'blob': 'http://localhost/testdocument/%s/blob' % guid}, + {'blob': {'url': 'http://localhost/testdocument/%s/blob' % guid}}, self.call('GET', path=['testdocument', guid], reply=['blob'], host='localhost')) def test_Command_ReplyForGET(self): -- cgit v0.9.1