diff options
Diffstat (limited to 'sugar_network')
| -rw-r--r-- | sugar_network/client/cache.py | 4 | ||||
| -rw-r--r-- | sugar_network/db/metadata.py | 48 | ||||
| -rw-r--r-- | sugar_network/db/routes.py | 32 | ||||
| -rw-r--r-- | sugar_network/model/__init__.py | 52 | ||||
| -rw-r--r-- | sugar_network/model/routes.py | 4 | ||||
| -rw-r--r-- | sugar_network/node/routes.py | 19 | ||||
| -rw-r--r-- | sugar_network/toolkit/__init__.py | 42 | ||||
| -rw-r--r-- | sugar_network/toolkit/http.py | 10 | ||||
| -rw-r--r-- | sugar_network/toolkit/router.py | 107 |
9 files changed, 145 insertions, 173 deletions
diff --git a/sugar_network/client/cache.py b/sugar_network/client/cache.py index df76a29..8bee316 100644 --- a/sugar_network/client/cache.py +++ b/sugar_network/client/cache.py @@ -13,6 +13,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +# sugar-lint: disable + import os import sys import time @@ -20,7 +22,7 @@ import logging from os.path import exists from sugar_network import client -from sugar_network.db import files +from sugar_network.db import blobs from sugar_network.toolkit import pylru, enforce diff --git a/sugar_network/db/metadata.py b/sugar_network/db/metadata.py index 5282fd1..88d644b 100644 --- a/sugar_network/db/metadata.py +++ b/sugar_network/db/metadata.py @@ -16,8 +16,8 @@ import xapian from sugar_network import toolkit -from sugar_network.db import files -from sugar_network.toolkit.router import ACL +from sugar_network.db import blobs +from sugar_network.toolkit.router import ACL, File from sugar_network.toolkit.coroutine import this from sugar_network.toolkit import i18n, http, enforce @@ -33,6 +33,7 @@ def stored_property(klass=None, *args, **kwargs): return func(self, value) def decorate_setter(func, attr): + # pylint: disable-msg=W0212 attr.prop.setter = lambda self, value: \ self._set(attr.name, func(self, value)) attr.prop.on_set = func @@ -297,12 +298,12 @@ class Blob(Property): self.mime_type = mime_type def typecast(self, value): - if isinstance(value, toolkit.File): + if isinstance(value, File): return value.digest - if isinstance(value, files.Digest): + if isinstance(value, File.Digest): return value - enforce(value is None or isinstance(value, basestring) or \ + enforce(value is None or isinstance(value, basestring) or isinstance(value, dict) and value or hasattr(value, 'read'), 'Inappropriate blob value') @@ -310,45 +311,38 @@ class Blob(Property): return '' if not isinstance(value, dict): - return files.post(value, { - 'mime_type': this.request.content_type or self.mime_type, - }).digest + mime_type = this.request.content_type or self.mime_type + return blobs.post(value, mime_type).digest digest = this.resource[self.name] if self.name else None if digest: - meta = files.get(digest) + orig = blobs.get(digest) enforce('digest' not in value or value.pop('digest') == digest, "Inappropriate 'digest' value") - enforce(meta.path or 'url' in meta or 'url' in value, + enforce(orig.path or 'location' in orig or 'location' in value, 'Blob points to nothing') - if 'url' in value and meta.path: - files.delete(digest) - meta.update(value) - value = meta + if 'location' in value and orig.path: + blobs.delete(digest) + orig.update(value) + value = orig else: - enforce('url' in value, 'Blob points to nothing') + enforce('location' in value, 'Blob points to nothing') enforce('digest' in value, "Missed 'digest' value") - if 'mime_type' not in value: - value['mime_type'] = self.mime_type + if 'content-type' not in value: + value['content-type'] = self.mime_type digest = value.pop('digest') - files.update(digest, value) + blobs.update(digest, value) return digest def reprcast(self, value): if not value: - return toolkit.File.AWAY - meta = files.get(value) - if 'url' not in meta: - meta['url'] = '%s/blobs/%s' % (this.request.static_prefix, value) - meta['size'] = meta.size - meta['mtime'] = meta.mtime - meta['digest'] = value - return meta + return File.AWAY + return blobs.get(value) def teardown(self, value): if value: - files.delete(value) + blobs.delete(value) def assert_access(self, mode, value=None): if mode == ACL.WRITE and not value: diff --git a/sugar_network/db/routes.py b/sugar_network/db/routes.py index 2f8fc69..d8d2fb4 100644 --- a/sugar_network/db/routes.py +++ b/sugar_network/db/routes.py @@ -15,14 +15,14 @@ import re import time -import json import logging from contextlib import contextmanager from sugar_network import toolkit -from sugar_network.db import files +from sugar_network.db import blobs from sugar_network.db.metadata import Aggregated -from sugar_network.toolkit.router import ACL, route, preroute, fallbackroute +from sugar_network.toolkit.router import ACL, File +from sugar_network.toolkit.router import route, preroute, fallbackroute from sugar_network.toolkit.coroutine import this from sugar_network.toolkit import http, enforce @@ -137,25 +137,20 @@ class Routes(object): prop = directory.metadata[request.prop] prop.assert_access(ACL.READ) - meta = doc.meta(prop.name) or {} - if 'value' in meta: - value = _get_prop(doc, prop, meta.pop('value')) - enforce(value is not toolkit.File.AWAY, http.NotFound, 'No blob') + meta = doc.meta(prop.name) + if meta: + value = meta['value'] + response.last_modified = meta['mtime'] else: value = prop.default - - response.meta = meta - response.last_modified = meta.get('mtime') - if isinstance(value, toolkit.File): - response.content_length = value.get('size') or 0 - else: - response.content_length = len(json.dumps(value)) + value = _get_prop(doc, prop, value) + enforce(value is not File.AWAY, http.NotFound, 'No blob') return value @route('HEAD', [None, None, None]) def get_prop_meta(self, request, response): - self.get_prop(request, response) + return self.get_prop(request, response) @route('POST', [None, None, None], acl=ACL.AUTH, mime_type='application/json') @@ -193,7 +188,7 @@ class Routes(object): @fallbackroute('GET', ['blobs']) def blobs(self, request): - return files.get(request.guid) + return blobs.get(request.guid) def on_create(self, request, props): ts = int(time.time()) @@ -280,7 +275,10 @@ class Routes(object): result = {} for name in props: prop = doc.metadata[name] - result[name] = _get_prop(doc, prop, doc.get(name)) + value = _get_prop(doc, prop, doc.get(name)) + if isinstance(value, File): + value = value.url + result[name] = value return result def _useradd(self, authors, user, role): diff --git a/sugar_network/model/__init__.py b/sugar_network/model/__init__.py index 6858957..f7be261 100644 --- a/sugar_network/model/__init__.py +++ b/sugar_network/model/__init__.py @@ -16,19 +16,20 @@ import os import gettext import logging +import mimetypes from os.path import join import xapian from sugar_network import toolkit, db -from sugar_network.db import files +from sugar_network.db import blobs from sugar_network.model.routes import FrontRoutes from sugar_network.toolkit.spec import parse_version, parse_requires from sugar_network.toolkit.spec import EMPTY_LICENSE from sugar_network.toolkit.coroutine import this from sugar_network.toolkit.bundle import Bundle from sugar_network.toolkit.router import ACL -from sugar_network.toolkit import i18n, http, exception, enforce +from sugar_network.toolkit import i18n, http, svg_to_png, exception, enforce CONTEXT_TYPES = [ @@ -73,22 +74,24 @@ class Rating(db.List): class Release(object): - def typecast(self, rel): + def typecast(self, release): if this.resource.exists and \ 'activity' not in this.resource['type'] and \ 'book' not in this.resource['type']: - return rel - if not isinstance(rel, dict): - __, rel = load_bundle(files.post(rel), context=this.request.guid) - return rel['spec']['*-*']['bundle'], rel - - def teardown(self, rel): + return release + if not isinstance(release, dict): + __, release = load_bundle( + blobs.post(release, this.request.content_type), + context=this.request.guid) + return release['spec']['*-*']['bundle'], release + + def teardown(self, release): if this.resource.exists and \ 'activity' not in this.resource['type'] and \ 'book' not in this.resource['type']: return - for spec in rel['spec'].values(): - files.delete(spec['bundle']) + for spec in release['spec'].values(): + blobs.delete(spec['bundle']) def encode(self, value): return [] @@ -120,18 +123,9 @@ def populate_context_images(props, svg): if 'guid' in props: from sugar_network.toolkit.sugar import color_svg svg = color_svg(svg, props['guid']) - props['artifact_icon'] = files.post( - svg, - {'mime_type': 'image/svg+xml'}, - ).digest - props['icon'] = files.post( - toolkit.svg_to_png(svg, 55, 55), - {'mime_type': 'image/png'}, - ).digest - props['logo'] = files.post( - toolkit.svg_to_png(svg, 140, 140), - {'mime_type': 'image/png'}, - ).digest + props['artifact_icon'] = blobs.post(svg, 'image/svg+xml').digest + props['icon'] = blobs.post(svg_to_png(svg, 55, 55), 'image/png').digest + props['logo'] = blobs.post(svg_to_png(svg, 140, 140), 'image/png').digest def load_bundle(blob, context=None, initial=False, extra_deps=None): @@ -140,7 +134,6 @@ def load_bundle(blob, context=None, initial=False, extra_deps=None): context_meta = None release_notes = None release = {} - blob_meta = {} version = None try: @@ -157,7 +150,6 @@ def load_bundle(blob, context=None, initial=False, extra_deps=None): release['spec'] = {'*-*': { 'bundle': blob.digest, }} - blob_meta['mime_type'] = this.request.content_type else: context_type = 'activity' unpack_size = 0 @@ -191,7 +183,7 @@ def load_bundle(blob, context=None, initial=False, extra_deps=None): 'bundle': blob.digest, }} release['unpack_size'] = unpack_size - blob_meta['mime_type'] = 'application/vnd.olpc-sugar' + blob['content-type'] = 'application/vnd.olpc-sugar' enforce(context, http.BadRequest, 'Context is not specified') enforce(version, http.BadRequest, 'Version is not specified') @@ -237,9 +229,11 @@ def load_bundle(blob, context=None, initial=False, extra_deps=None): }, content_type='application/json') - filename = ''.join(i18n.decode(context_doc['title']).split()) - blob_meta['name'] = '%s-%s' % (filename, version) - files.update(blob.digest, blob_meta) + blob['content-disposition'] = 'attachment; filename="%s-%s%s"' % ( + ''.join(i18n.decode(context_doc['title']).split()), + version, mimetypes.guess_extension(blob.get('content-type')) or '', + ) + blobs.update(blob.digest, blob) return context, release diff --git a/sugar_network/model/routes.py b/sugar_network/model/routes.py index ff0377f..35c56a9 100644 --- a/sugar_network/model/routes.py +++ b/sugar_network/model/routes.py @@ -15,7 +15,7 @@ import logging -from sugar_network.db import files +from sugar_network.db import blobs from sugar_network.toolkit.router import route from sugar_network.toolkit.coroutine import this from sugar_network.toolkit import coroutine @@ -61,7 +61,7 @@ class FrontRoutes(object): @route('GET', ['favicon.ico']) def favicon(self, request, response): - return files.get('favicon.ico') + return blobs.get('favicon.ico') def _broadcast(self, event): _logger.debug('Broadcast event: %r', event) diff --git a/sugar_network/node/routes.py b/sugar_network/node/routes.py index da6c675..66b7823 100644 --- a/sugar_network/node/routes.py +++ b/sugar_network/node/routes.py @@ -21,7 +21,7 @@ from ConfigParser import ConfigParser from os.path import join, isdir, exists from sugar_network import db, node, toolkit -from sugar_network.db import files +from sugar_network.db import blobs from sugar_network.model import FrontRoutes, load_bundle from sugar_network.node import stats_user, model # pylint: disable-msg=W0611 @@ -119,11 +119,11 @@ class NodeRoutes(db.Routes, FrontRoutes): arguments={'initial': False}, mime_type='application/json', acl=ACL.AUTH) def submit_release(self, request, initial): - blob = files.post(request.content_stream) + blob = blobs.post(request.content_stream, request.content_type) try: context, release = load_bundle(blob, initial=initial) except Exception: - files.delete(blob.digest) + blobs.delete(blob.digest) raise this.call(method='POST', path=['context', context, 'releases'], content_type='application/json', content=release) @@ -156,13 +156,8 @@ class NodeRoutes(db.Routes, FrontRoutes): @route('GET', ['context', None], cmd='clone', arguments={'requires': list}) def get_clone(self, request, response): - response.meta = self.solve(request) - return files.get(response.meta['files'][request.guid]) - - @route('HEAD', ['context', None], cmd='clone', - arguments={'requires': list}) - def head_clone(self, request, response): - response.meta = self.solve(request) + solution = self.solve(request) + return blobs.get(solution['files'][request.guid]) @route('GET', ['user', None], cmd='stats-info', mime_type='application/json', acl=ACL.AUTH) @@ -210,7 +205,7 @@ class NodeRoutes(db.Routes, FrontRoutes): def on_create(self, request, props): if request.resource == 'user': - with file(files.get(props['pubkey']).path) as f: + with file(blobs.get(props['pubkey']).path) as f: props['guid'] = str(hashlib.sha1(f.read()).hexdigest()) db.Routes.on_create(self, request, props) @@ -229,7 +224,7 @@ class NodeRoutes(db.Routes, FrontRoutes): from M2Crypto import RSA pubkey = self.volume['user'][auth.login]['pubkey'] - key = RSA.load_pub_key(files.get(pubkey).path) + key = RSA.load_pub_key(blobs.get(pubkey).path) data = hashlib.sha1('%s:%s' % (auth.login, auth.nonce)).digest() enforce(key.verify(data, auth.signature.decode('hex')), http.Forbidden, 'Bad credentials') diff --git a/sugar_network/toolkit/__init__.py b/sugar_network/toolkit/__init__.py index 4088e07..073ec4d 100644 --- a/sugar_network/toolkit/__init__.py +++ b/sugar_network/toolkit/__init__.py @@ -450,45 +450,6 @@ def svg_to_png(data, w, h): return result -class File(dict): - - AWAY = None - - def __init__(self, path=None, meta=None, digest=None): - self.path = path - self.digest = digest - dict.__init__(self, meta or {}) - self._stat = None - self._name = self.get('filename') - - @property - def size(self): - if self._stat is None: - self._stat = os.stat(self.path) - return self._stat.st_size - - @property - def mtime(self): - if self._stat is None: - self._stat = os.stat(self.path) - return int(self._stat.st_mtime) - - @property - def name(self): - if self._name is None: - self._name = self.get('name') or self.digest or 'blob' - mime_type = self.get('mime_type') - if mime_type: - import mimetypes - if not mimetypes.inited: - mimetypes.init() - self._name += mimetypes.guess_extension(mime_type) or '' - return self._name - - def __repr__(self): - return '<File path=%r digest=%r>' % (self.path, self.digest) - - def TemporaryFile(*args, **kwargs): if 'dir' not in kwargs: kwargs['dir'] = cachedir.value @@ -843,6 +804,3 @@ def _nb_read(stream): return '' finally: fcntl.fcntl(fd, fcntl.F_SETFL, orig_flags) - - -File.AWAY = File() diff --git a/sugar_network/toolkit/http.py b/sugar_network/toolkit/http.py index 8d913ae..47f13bc 100644 --- a/sugar_network/toolkit/http.py +++ b/sugar_network/toolkit/http.py @@ -142,7 +142,7 @@ class Connection(object): request = Request(method='HEAD', path=path_, **kwargs) response = Response() self.call(request, response) - return response.meta + return response def get(self, path_=None, query_=None, **kwargs): reply = self.request('GET', path_, params=query_ or kwargs) @@ -274,13 +274,11 @@ class Connection(object): if 'transfer-encoding' in reply.headers: # `requests` library handles encoding on its own del reply.headers['transfer-encoding'] - for key, value in reply.headers.items(): - if key.startswith('x-sn-'): - response.meta[key[5:]] = json.loads(value) - elif not resend: - response[key] = value if resend: response.relocations += 1 + else: + for key, value in reply.headers.items(): + response[key] = value if not resend: break path = reply.headers['location'] diff --git a/sugar_network/toolkit/router.py b/sugar_network/toolkit/router.py index b37eee4..00e5d8a 100644 --- a/sugar_network/toolkit/router.py +++ b/sugar_network/toolkit/router.py @@ -346,15 +346,35 @@ class Request(dict): (self.method, self.path, self.cmd, dict(self)) -class Response(dict): +class CaseInsensitiveDict(dict): + + def __contains__(self, key): + return dict.__contains__(self, key.lower()) + + def __getitem__(self, key): + return self.get(key.lower()) + + def __setitem__(self, key, value): + return self.set(key.lower(), value) + + def __delitem__(self, key, value): + self.remove(key.lower()) + + def get(self, key): + return dict.get(self, key) + + def set(self, key, value): + dict.__setitem__(self, key, value) + + def remove(self, key): + dict.__delitem__(self, key) + + +class Response(CaseInsensitiveDict): status = '200 OK' relocations = 0 - def __init__(self, **kwargs): - dict.__init__(self, kwargs) - self.meta = {} - @property def content_length(self): return int(self.get('content-length') or '0') @@ -394,26 +414,47 @@ class Response(dict): return result def __repr__(self): - items = ['%s=%r' % i for i in self.items() + self.meta.items()] + items = ['%s=%r' % i for i in self.items()] return '<Response %r>' % items - def __contains__(self, key): - dict.__contains__(self, key.lower()) - def __getitem__(self, key): - return self.get(key.lower()) +class File(CaseInsensitiveDict): - def __setitem__(self, key, value): - return self.set(key.lower(), value) + AWAY = None - def __delitem__(self, key, value): - self.remove(key.lower()) + class Digest(str): + pass - def set(self, key, value): - dict.__setitem__(self, key, value) + def __init__(self, path, digest=None, meta=None): + CaseInsensitiveDict.__init__(self) + self.path = path + self.digest = File.Digest(digest) if digest else None + if meta is not None: + for key, value in meta: + self[key] = value + self._stat = None - def remove(self, key): - dict.__delitem__(self, key) + @property + def size(self): + if self._stat is None: + self._stat = os.stat(self.path) + return self._stat.st_size + + @property + def mtime(self): + if self._stat is None: + self._stat = os.stat(self.path) + return int(self._stat.st_mtime) + + @property + def url(self): + if self is File.AWAY: + return '' + return self.get('location') or \ + '%s/blobs/%s' % (this.request.static_prefix, self.digest) + + def __repr__(self): + return '<File %r>' % self.url class Router(object): @@ -530,11 +571,8 @@ class Router(object): except Exception, exception: raise else: - if not response.content_type: - if isinstance(result, toolkit.File): - response.content_type = result.get('mime_type') - if not response.content_type: - response.content_type = route_.mime_type + if route_.mime_type and 'content-type' not in response: + response.set('content-type', route_.mime_type) finally: for i in self._postroutes: i(request, response, result, exception) @@ -563,18 +601,14 @@ class Router(object): result = self.call(request, response) - if isinstance(result, toolkit.File): - if 'url' in result: - raise http.Redirect(result['url']) + if isinstance(result, File): + response.update(result) + if 'location' in result: + raise http.Redirect(result['location']) enforce(isfile(result.path), 'No such file') - if request.if_modified_since and result.mtime and \ + if request.if_modified_since and \ result.mtime <= request.if_modified_since: raise http.NotModified() - response.last_modified = result.mtime - response.content_type = result.get('mime_type') or \ - 'application/octet-stream' - response['Content-Disposition'] = \ - 'attachment; filename="%s"' % result.name result = file(result.path, 'rb') if not hasattr(result, 'read'): @@ -592,7 +626,6 @@ class Router(object): response.status = error.status if error.headers: response.update(error.headers) - response.content_type = None except Exception, error: toolkit.exception('Error while processing %r request', request.url) if isinstance(error, http.Status): @@ -601,7 +634,7 @@ class Router(object): else: response.status = '500 Internal Server Error' if request.method == 'HEAD': - response.meta['error'] = str(error) + response.status = response.status[:4] + str(error) else: content = {'error': str(error), 'request': request.url} response.content_type = 'application/json' @@ -623,9 +656,6 @@ class Router(object): if 'content-length' not in response: response.content_length = len(content) if content else 0 - for key, value in response.meta.items(): - response.set('X-SN-%s' % toolkit.ascii(key), json.dumps(value)) - if request.method == 'HEAD' and content is not None: _logger.warning('Content from HEAD response is ignored') content = None @@ -859,3 +889,6 @@ class _Authorization(str): password = None signature = None nonce = None + + +File.AWAY = File(None) |