diff options
| author | Aleksey Lim <alsroot@sugarlabs.org> | 2013-07-27 17:25:48 (GMT) |
|---|---|---|
| committer | Aleksey Lim <alsroot@sugarlabs.org> | 2013-07-27 17:25:48 (GMT) |
| commit | fa35499bcd89260b436c3c9f03c887661983c14d (patch) | |
| tree | 7401fbbbbefce5885bb1fb5b52bbc17f8c0dfe72 | |
| parent | 35d777deda94fbc78324c447e16bb7abf0b11434 (diff) | |
Polish implementation
106 files changed, 6850 insertions, 7335 deletions
@@ -1,3 +1,6 @@ +- 0.10 + - s/implementation/version/ + - delete outdated impls on PUTing new one - (!) Editors' workflows: diff --git a/doc/objects.dia b/doc/objects.dia index 9444a06..c010b7a 100644 --- a/doc/objects.dia +++ b/doc/objects.dia @@ -523,25 +523,25 @@ </dia:object> <dia:object type="UML - Class" version="0" id="O1"> <dia:attribute name="obj_pos"> - <dia:point val="-2,6"/> + <dia:point val="3,6"/> </dia:attribute> <dia:attribute name="obj_bb"> - <dia:rectangle val="-2.015,5.985;9.5475,33.615"/> + <dia:rectangle val="2.985,5.985;14.5475,28.015"/> </dia:attribute> <dia:attribute name="elem_corner"> - <dia:point val="-2,6"/> + <dia:point val="3,6"/> </dia:attribute> <dia:attribute name="elem_width"> <dia:real val="11.532500000000001"/> </dia:attribute> <dia:attribute name="elem_height"> - <dia:real val="27.600000000000001"/> + <dia:real val="22.000000000000004"/> </dia:attribute> <dia:attribute name="name"> <dia:string>#User#</dia:string> </dia:attribute> <dia:attribute name="stereotype"> - <dia:string>##</dia:string> + <dia:string>#Resource#</dia:string> </dia:attribute> <dia:attribute name="comment"> <dia:string>#Represents real people#</dia:string> @@ -630,29 +630,6 @@ <dia:attribute name="attributes"> <dia:composite type="umlattribute"> <dia:attribute name="name"> - <dia:string>#guid#</dia:string> - </dia:attribute> - <dia:attribute name="type"> - <dia:string>#str [R]#</dia:string> - </dia:attribute> - <dia:attribute name="value"> - <dia:string>##</dia:string> - </dia:attribute> - <dia:attribute name="comment"> - <dia:string>#Hashed value of Sugar profile public SSH key (the same as JID value in Sugar Shell but without the domain part)#</dia:string> - </dia:attribute> - <dia:attribute name="visibility"> - <dia:enum val="0"/> - </dia:attribute> - <dia:attribute name="abstract"> - <dia:boolean val="false"/> - </dia:attribute> - <dia:attribute name="class_scope"> - <dia:boolean val="false"/> - </dia:attribute> - </dia:composite> - <dia:composite type="umlattribute"> - <dia:attribute name="name"> <dia:string>#pubkey#</dia:string> </dia:attribute> <dia:attribute name="type"> @@ -722,29 +699,6 @@ </dia:composite> <dia:composite type="umlattribute"> <dia:attribute name="name"> - <dia:string>#tags#</dia:string> - </dia:attribute> - <dia:attribute name="type"> - <dia:string>#[str] [R WA F]#</dia:string> - </dia:attribute> - <dia:attribute name="value"> - <dia:string>#[]#</dia:string> - </dia:attribute> - <dia:attribute name="comment"> - <dia:string>#List of tags#</dia:string> - </dia:attribute> - <dia:attribute name="visibility"> - <dia:enum val="0"/> - </dia:attribute> - <dia:attribute name="abstract"> - <dia:boolean val="false"/> - </dia:attribute> - <dia:attribute name="class_scope"> - <dia:boolean val="false"/> - </dia:attribute> - </dia:composite> - <dia:composite type="umlattribute"> - <dia:attribute name="name"> <dia:string>#machine_sn#</dia:string> </dia:attribute> <dia:attribute name="type"> @@ -844,13 +798,13 @@ </dia:object> <dia:object type="UML - Class" version="0" id="O2"> <dia:attribute name="obj_pos"> - <dia:point val="12,6"/> + <dia:point val="-13,6"/> </dia:attribute> <dia:attribute name="obj_bb"> - <dia:rectangle val="11.985,5.985;23.77,26.015"/> + <dia:rectangle val="-13.015,5.985;-1.23,26.015"/> </dia:attribute> <dia:attribute name="elem_corner"> - <dia:point val="12,6"/> + <dia:point val="-13,6"/> </dia:attribute> <dia:attribute name="elem_width"> <dia:real val="11.754999999999999"/> diff --git a/sugar-network b/sugar-network index bd0a6d6..d5adbc3 100755 --- a/sugar-network +++ b/sugar-network @@ -19,6 +19,7 @@ import os import re import sys import shlex +import types import locale from json import dumps, loads from os.path import join, exists @@ -26,9 +27,9 @@ from os.path import join, exists from gevent import monkey from sugar_network import db, client, toolkit -from sugar_network.resources.volume import Volume -from sugar_network.client import IPCRouter -from sugar_network.client.commands import ClientCommands +from sugar_network.model import RESOURCES +from sugar_network.client.routes import ClientRoutes +from sugar_network.toolkit.router import Router, Request, Response from sugar_network.toolkit import application, coroutine, util from sugar_network.toolkit import Option, BUFFER_SIZE, enforce @@ -68,6 +69,25 @@ uri = Option( _ESCAPE_VALUE_RE = re.compile('([^\\[\\]\\{\\}0-9][^\\]\\[\\{\\}]+)') +class Client(ClientRoutes, Router): + + def __init__(self): + home = db.Volume(client.path('db'), RESOURCES) + ClientRoutes.__init__(self, home, + client.api_url.value if not offline.value else None, + no_subscription=True) + Router.__init__(self, self) + + if not offline.value: + for __ in self.subscribe(event='inline', state='online'): + break + coroutine.dispatch() + server = coroutine.WSGIServer( + ('localhost', client.ipc_port.value), self) + coroutine.spawn(server.serve_forever) + coroutine.dispatch() + + class Application(application.Application): def __init__(self, **kwargs): @@ -81,7 +101,7 @@ class Application(application.Application): @application.command( 'launch Sugar activity', - args='BUNDLE_ID [COMMAND-LINE-ARGS]', + args='BUNDLE_ID [COMMAND-LINE-ARGS]', interspersed_args=False, ) def launch(self): enforce(self.check_for_instance(), 'No sugar-network-client session') @@ -134,9 +154,10 @@ class Application(application.Application): self._request('GET', False) def _request(self, method, post): - request = db.Request(method=method) + request = Request(method=method) request.allow_redirects = True - response = db.Response() + request.accept_encoding = '' + response = Response() reply = [] if post: @@ -161,12 +182,7 @@ class Application(application.Application): pass if self.args and self.args[0].startswith('/'): - path = self.args.pop(0).strip('/').split('/') - request['document'] = path.pop(0) - if path: - request['guid'] = path.pop(0) - if path: - request['prop'] = path.pop(0) + request.path = self.args.pop(0).strip('/').split('/') for arg in self.args: arg = shlex.split(arg) @@ -196,18 +212,7 @@ class Application(application.Application): pid_path = self.new_instance() if not client.anonymous.value: util.ensure_key(client.key_path()) - home = Volume(client.path('db')) - cp = ClientCommands(home, - client.api_url.value if not offline.value else None, - no_subscription=True) - if not offline.value: - for __ in cp.subscribe(event='inline', state='online'): - break - coroutine.dispatch() - server = coroutine.WSGIServer( - ('localhost', client.ipc_port.value), IPCRouter(cp)) - coroutine.spawn(server.serve_forever) - coroutine.dispatch() + cp = Client() result = cp.call(request, response) finally: if server is not None: @@ -247,6 +252,9 @@ class Application(application.Application): if not chunk: break sys.stdout.write(chunk) + elif isinstance(result, types.GeneratorType): + for chunk in result: + sys.stdout.write(chunk) else: sys.stdout.write(result) @@ -284,5 +292,5 @@ app = Application( '~/.config/sweets/config', client.profile_path('sweets.conf'), ], - stop_args=['launch']) + ) app.start() diff --git a/sugar-network-client b/sugar-network-client index 0efd7dc..b034d0a 100755 --- a/sugar-network-client +++ b/sugar-network-client @@ -26,10 +26,11 @@ from gevent import monkey import sugar_network_webui as webui from sugar_network import db, toolkit, client, node -from sugar_network.client import IPCRouter, clones, cache -from sugar_network.client.commands import CachedClientCommands +from sugar_network.client import clones, cache +from sugar_network.client.routes import CachedClientRoutes from sugar_network.node import stats_node, stats_user -from sugar_network.resources.volume import Volume +from sugar_network.model import RESOURCES +from sugar_network.toolkit.router import Router from sugar_network.toolkit import mountpoints, util, printf, application from sugar_network.toolkit import Option, coroutine @@ -69,7 +70,7 @@ class Application(application.Daemon): printf.info('Index database in %r', client.local_root.value) - volume = Volume(client.path('db')) + volume = db.Volume(client.path('db'), RESOURCES) try: volume.populate() clones.populate(volume['context'], client.activity_dirs.value) @@ -104,14 +105,14 @@ class Application(application.Daemon): def run(self): util.ensure_key(client.key_path()) - volume = Volume(client.path('db')) - commands = CachedClientCommands(volume, + volume = db.Volume(client.path('db'), RESOURCES) + routes = CachedClientRoutes(volume, client.api_url.value if not client.server_mode.value else None) logging.info('Listening for IPC requests on %s port', client.ipc_port.value) server = coroutine.WSGIServer(('localhost', client.ipc_port.value), - IPCRouter(commands)) + Router(routes)) self.jobs.spawn(server.serve_forever) coroutine.dispatch() @@ -125,7 +126,7 @@ class Application(application.Daemon): if webui.webui.value: host = (webui.webui_host.value, webui.webui_port.value) logging.info('Start Web server on %s:%s', *host) - server = coroutine.WSGIServer(host, webui.get_app(commands, + server = coroutine.WSGIServer(host, webui.get_app(routes, 'http://localhost:%s' % client.ipc_port.value)) self.jobs.spawn(server.serve_forever) @@ -139,7 +140,7 @@ class Application(application.Daemon): self.jobs.spawn(self._recycle_cache) def delayed_start(event=None): - for __ in commands.subscribe(event='delayed-start'): + for __ in routes.subscribe(event='delayed-start'): break logging.info('Proceed delayed start') final_start() @@ -155,7 +156,7 @@ class Application(application.Daemon): util.exception('%s interrupted', self.name) finally: self.jobs.kill() - commands.close() + routes.close() def shutdown(self): self.jobs.kill() diff --git a/sugar-network-node b/sugar-network-node index c319672..681c8f2 100755 --- a/sugar-network-node +++ b/sugar-network-node @@ -24,12 +24,12 @@ from gevent import monkey import sugar_network_webui as webui from sugar_network import db, node, client, toolkit -from sugar_network.db.router import Router from sugar_network.node import stats_node, stats_user, obs -from sugar_network.node.master import MasterCommands -from sugar_network.node.slave import SlaveCommands -from sugar_network.resources.volume import Volume +from sugar_network.node.master import MasterRoutes +from sugar_network.node.slave import SlaveRoutes +from sugar_network.model import RESOURCES from sugar_network.toolkit.http import Client +from sugar_network.toolkit.router import Router from sugar_network.toolkit import coroutine, application, util, Option, enforce @@ -55,7 +55,7 @@ class Application(application.Daemon): if node.certfile.value: ssl_args['certfile'] = node.certfile.value - volume = Volume(node.data_root.value) + volume = db.Volume(node.data_root.value, RESOURCES) self.jobs.spawn(volume.populate) master_path = join(node.data_root.value, 'master') @@ -63,10 +63,10 @@ class Application(application.Daemon): with file(master_path) as f: guid = f.read().strip() logging.info('Start %s node in master mode', guid) - cp = MasterCommands(guid, volume) + cp = MasterRoutes(guid, volume) else: logging.info('Start slave node') - cp = SlaveCommands(join(node.data_root.value, 'node.key'), volume) + cp = SlaveRoutes(join(node.data_root.value, 'node.key'), volume) logging.info('Listening for requests on %s:%s', node.host.value, node.port.value) @@ -88,8 +88,8 @@ class Application(application.Daemon): client.sugar_uid = lambda: 'demo' # Point client API to volume directly client.mounts_root.value = None - home_volume = Volume(join(application.rundir.value, 'db')) - client_commands = _ClientCommands(home_volume, + home = db.Volume(join(application.rundir.value, 'db'), RESOURCES) + client_commands = _ClientCommands(home, api_url='http://localhost:%s' % node.port.value, static_prefix=client.api_url.value) host = (node.host.value, webui.webui_port.value) diff --git a/sugar_network/client/__init__.py b/sugar_network/client/__init__.py index c3d70ab..2c4f23b 100644 --- a/sugar_network/client/__init__.py +++ b/sugar_network/client/__init__.py @@ -17,7 +17,8 @@ import os import logging from os.path import join, expanduser, exists -from sugar_network.toolkit import Option, util +from sugar_network import toolkit +from sugar_network.toolkit import Option SUGAR_API_COMPATIBILITY = { @@ -123,9 +124,7 @@ anonymous = Option( name='anonymous') accept_language = Option( - 'space separated list of languages to request localized content ' - 'from the server; by default, reuse system locale', - default=[], type_cast=Option.list_cast, type_repr=Option.list_repr, + 'specify HTTP Accept-Language header field value manually', name='accept-language', short_option='-l') cache_limit = Option( @@ -192,22 +191,6 @@ def IPCClient(): ) -def IPCRouter(*args, **kwargs): - from sugar_network import db - from sugar_network.db.router import Router - - class _IPCRouter(Router): - - def authenticate(self, request): - pass - - def call(self, request, response): - request.access_level = db.ACCESS_LOCAL - return Router.call(self, request, response) - - return _IPCRouter(*args, **kwargs) - - def logger_level(): """Current Sugar logger level as --debug value.""" _LEVELS = { @@ -229,7 +212,7 @@ def sugar_uid(): global _sugar_uid if _sugar_uid is None: import hashlib - pubkey = util.pubkey(key_path()).split()[1] + pubkey = toolkit.pubkey(key_path()).split()[1] _sugar_uid = str(hashlib.sha1(pubkey).hexdigest()) return _sugar_uid @@ -241,7 +224,7 @@ def sugar_profile(): 'color': conf.get_string(_COLOR_GCONF) or '#000000,#000000', 'machine_sn': _read_XO_value(_XO_SERIAL_PATH) or '', 'machine_uuid': _read_XO_value(_XO_UUID_PATH) or '', - 'pubkey': util.pubkey(key_path()), + 'pubkey': toolkit.pubkey(key_path()), } diff --git a/sugar_network/client/cache.py b/sugar_network/client/cache.py index 6c91b17..1d8167f 100644 --- a/sugar_network/client/cache.py +++ b/sugar_network/client/cache.py @@ -20,10 +20,11 @@ import shutil import logging from os.path import exists, join, isdir +from sugar_network import toolkit from sugar_network.client import IPCClient, local_root from sugar_network.client import cache_limit, cache_lifetime from sugar_network.toolkit.bundle import Bundle -from sugar_network.toolkit import pipe, util, exception, enforce +from sugar_network.toolkit import pipe, enforce _logger = logging.getLogger('cache') @@ -92,7 +93,7 @@ def get(guid, hints=None): ensure(hints.get('unpack_size') or 0, hints.get('bundle_size') or 0) blob = IPCClient().download(['implementation', guid, 'data']) _unpack_stream(blob, path) - with util.new_file(join(path, '.unpack_size')) as f: + with toolkit.new_file(join(path, '.unpack_size')) as f: json.dump(hints.get('unpack_size') or 0, f) topdir = os.listdir(path)[-1:] @@ -122,13 +123,13 @@ def _list(): # Negative `unpack_size` to process large impls at first result.append((os.stat(path).st_mtime, -unpack_size, path)) except Exception: - exception('Cannot list %r cached implementation', path) + toolkit.exception('Cannot list %r cached implementation', path) result.append((0, 0, path)) return total, sorted(result) def _unpack_stream(stream, dst): - with util.NamedTemporaryFile() as tmp_file: + with toolkit.NamedTemporaryFile() as tmp_file: for chunk in stream: tmp_file.write(chunk) tmp_file.flush() diff --git a/sugar_network/client/clones.py b/sugar_network/client/clones.py index 07114f3..34ff8c6 100644 --- a/sugar_network/client/clones.py +++ b/sugar_network/client/clones.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Aleksey Lim +# Copyright (C) 2012-2013 Aleksey Lim # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -21,12 +21,12 @@ import logging from os.path import join, exists, lexists, relpath, dirname, basename, isdir from os.path import abspath, islink -from sugar_network import db, client +from sugar_network import db, client, toolkit from sugar_network.toolkit.spec import Spec from sugar_network.toolkit.inotify import Inotify, \ IN_DELETE_SELF, IN_CREATE, IN_DELETE, IN_CLOSE_WRITE, \ IN_MOVED_TO, IN_MOVED_FROM -from sugar_network.toolkit import coroutine, util, exception +from sugar_network.toolkit import coroutine _logger = logging.getLogger('client.clones') @@ -123,7 +123,7 @@ class _Inotify(Inotify): try: cb(filename, event) except Exception: - exception('Cannot dispatch 0x%X event for %r', + toolkit.exception('Cannot dispatch 0x%X event for %r', event, filename) coroutine.dispatch() @@ -137,7 +137,7 @@ class _Inotify(Inotify): try: spec = Spec(root=clone_path) except Exception: - exception(_logger, 'Cannot read %r spec', clone_path) + toolkit.exception(_logger, 'Cannot read %r spec', clone_path) return context = spec['context'] @@ -171,8 +171,8 @@ class _Inotify(Inotify): with file(icon_path, 'rb') as f: self._contexts.update(context, {'artifact_icon': {'blob': f}}) - with util.NamedTemporaryFile() as f: - util.svg_to_png(icon_path, f.name, 32, 32) + with toolkit.NamedTemporaryFile() as f: + toolkit.svg_to_png(icon_path, f.name, 32, 32) self._contexts.update(context, {'icon': {'blob': f.name}}) self._checkin_activity(spec) @@ -187,8 +187,8 @@ class _Inotify(Inotify): _logger.debug('Update MIME database to process found %r', src_path) - util.symlink(src_path, dst_path) - util.spawn('update-mime-database', self._mime_dir) + toolkit.symlink(src_path, dst_path) + toolkit.spawn('update-mime-database', self._mime_dir) def lost(self, clone_path): __, checkin_path = _checkin_path(clone_path) @@ -222,7 +222,7 @@ class _Inotify(Inotify): _logger.debug('Update MIME database to process lost %r', impl_path) os.unlink(dst_path) - util.spawn('update-mime-database', self._mime_dir) + toolkit.spawn('update-mime-database', self._mime_dir) def _checkin_activity(self, spec): icon_path = join(spec.root, spec['icon']) @@ -232,7 +232,7 @@ class _Inotify(Inotify): if not exists(self._icons_dir): os.makedirs(self._icons_dir) for mime_type in spec['mime_types']: - util.symlink(icon_path, + toolkit.symlink(icon_path, join(self._icons_dir, mime_type.replace('/', '-') + '.svg')) diff --git a/sugar_network/client/injector.py b/sugar_network/client/injector.py index 02c7937..b7bfd83 100644 --- a/sugar_network/client/injector.py +++ b/sugar_network/client/injector.py @@ -1,4 +1,4 @@ -# Copyright (C) 2010-2012 Aleksey Lim +# Copyright (C) 2010-2013 Aleksey Lim # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,9 +19,9 @@ import shutil import logging from os.path import join, exists, basename, dirname -from sugar_network import client +from sugar_network import client, toolkit from sugar_network.client import journal, cache -from sugar_network.toolkit import pipe, lsb_release, util +from sugar_network.toolkit import pipe, lsb_release _PMS_PATHS = { @@ -134,11 +134,11 @@ def _clone(context): if not path or \ path == '/': # Fake path set by "sugar" dependency continue - dst_path = util.unique_filename( + dst_path = toolkit.unique_filename( client.activity_dirs.value[0], basename(path)) cloned.append(dst_path) _logger.info('Clone implementation to %r', dst_path) - util.cptree(path, dst_path) + toolkit.cptree(path, dst_path) impl['path'] = dst_path except Exception: while cloned: @@ -157,11 +157,11 @@ def _clone_impl(context_guid, params): src_path = cache.get(impl['guid'], impl) if 'extract' in impl: src_path = join(src_path, impl['extract']) - dst_path = util.unique_filename( + dst_path = toolkit.unique_filename( client.activity_dirs.value[0], basename(src_path)) _logger.info('Clone implementation to %r', dst_path) - util.cptree(src_path, dst_path) + toolkit.cptree(src_path, dst_path) _set_cached_solution(context_guid, [{ 'id': dst_path, diff --git a/sugar_network/client/journal.py b/sugar_network/client/journal.py index 6987f45..8f6c023 100644 --- a/sugar_network/client/journal.py +++ b/sugar_network/client/journal.py @@ -23,7 +23,8 @@ import logging from shutil import copyfileobj from tempfile import NamedTemporaryFile -from sugar_network import db, client +from sugar_network import client +from sugar_network.toolkit.router import Blob, route, Request from sugar_network.toolkit import enforce @@ -51,7 +52,7 @@ def get(guid, prop): return f.read() -class Commands(object): +class Routes(object): _ds = None @@ -68,43 +69,93 @@ class Commands(object): 'Cannot connect to sugar-datastore, ' 'Journal integration is disabled') - @db.route('GET', '/journal') - def journal(self, request, response): + @route('GET', ['journal'], mime_type='application/json', arguments={ + 'offset': int, + 'limit': int, + 'reply': ('uid', 'title', 'description', 'preview'), + 'order_by': list, + }) + def journal_find(self, request, response): enforce(self._ds is not None, 'Journal is inaccessible') - enforce(len(request.path) <= 3, 'Invalid request') - if len(request.path) == 1: - return self._find(request, response) - elif len(request.path) == 2: - return self._get(request, response) - elif len(request.path) == 3: - return self._get_prop(request, response) + import dbus + + reply = request.pop('reply') + if 'preview' in reply: + reply.remove('preview') + has_preview = True + else: + has_preview = False + for key in ('timestamp', 'filesize', 'creation_time'): + value = request.get(key) + if not value or '..' not in value: + continue + start, end = value.split('..', 1) + value = {'start': start or '0', 'end': end or str(sys.maxint)} + request[key] = dbus.Dictionary(value) + if 'uid' not in reply: + reply.append('uid') + + result, total = self._ds.find(request, reply, byte_arrays=True) - @db.route('PUT', '/journal') + for item in result: + # Do not break SN like API + guid = item['guid'] = item.pop('uid') + if has_preview: + item['preview'] = _preview_url(guid) + + return {'result': result, 'total': int(total)} + + @route('GET', ['journal', None], mime_type='application/json') + def journal_get(self, request, response): + guid = request.guid + return {'guid': guid, + 'title': get(guid, 'title'), + 'description': get(guid, 'description'), + 'preview': _preview_url(guid), + } + + @route('GET', ['journal', None, 'preview']) + def journal_get_preview(self, request, response): + return Blob({ + 'blob': _prop_path(request.guid, 'preview'), + 'mime_type': 'image/png', + }) + + @route('GET', ['journal', None, 'data']) + def journal_get_data(self, request, response): + return Blob({ + 'blob': _ds_path(request.guid, 'data'), + 'mime_type': get(request.guid, 'mime_type') or 'application/octet', + }) + + @route('GET', ['journal', None, None], mime_type='application/json') + def journal_get_prop(self, request, response): + return get(request.guid, request.prop) + + @route('PUT', ['journal', None], cmd='share') def journal_share(self, request, response): enforce(self._ds is not None, 'Journal is inaccessible') - enforce(len(request.path) == 2 and request.get('cmd') == 'share', - 'Invalid request') - guid = request.path[1] + guid = request.guid preview_path = _prop_path(guid, 'preview') enforce(os.access(preview_path, os.R_OK), 'No preview') data_path = _ds_path(guid, 'data') enforce(os.access(data_path, os.R_OK), 'No data') - subrequest = db.Request(method='POST', document='artifact') + subrequest = Request(method='POST', document='artifact') subrequest.content = request.content subrequest.content_type = 'application/json' # pylint: disable-msg=E1101 subguid = self.call(subrequest, response) - subrequest = db.Request(method='PUT', document='artifact', + subrequest = Request(method='PUT', document='artifact', guid=subguid, prop='preview') subrequest.content_type = 'image/png' with file(preview_path, 'rb') as subrequest.content_stream: self.call(subrequest, response) - subrequest = db.Request(method='PUT', document='artifact', + subrequest = Request(method='PUT', document='artifact', guid=subguid, prop='data') subrequest.content_type = get(guid, 'mime_type') or 'application/octet' with file(data_path, 'rb') as subrequest.content_stream: @@ -145,67 +196,6 @@ class Commands(object): enforce(self._ds is not None, 'Journal is inaccessible') self._ds.delete(guid) - def _find(self, request, response): - import dbus - - if 'order_by' in request: - request['order_by'] = [request['order_by']] - for key in ('offset', 'limit'): - if key in request: - request[key] = int(request[key]) - if 'reply' in request: - reply = db.to_list(request.pop('reply')) - else: - reply = ['uid', 'title', 'description', 'preview'] - if 'preview' in reply: - reply.remove('preview') - has_preview = True - else: - has_preview = False - for key in ('timestamp', 'filesize', 'creation_time'): - value = request.get(key) - if not value or '..' not in value: - continue - start, end = value.split('..', 1) - value = {'start': start or '0', 'end': end or str(sys.maxint)} - request[key] = dbus.Dictionary(value) - if 'uid' not in reply: - reply.append('uid') - - result, total = self._ds.find(request, reply, byte_arrays=True) - - for item in result: - # Do not break SN like API - guid = item['guid'] = item.pop('uid') - if has_preview: - item['preview'] = _preview_url(guid) - - response.content_type = 'application/json' - return {'result': result, 'total': int(total)} - - def _get(self, request, response): - guid = request.path[1] - response.content_type = 'application/json' - return {'guid': guid, - 'title': get(guid, 'title'), - 'description': get(guid, 'description'), - 'preview': _preview_url(guid), - } - - def _get_prop(self, request, response): - guid = request.path[1] - prop = request.path[2] - - if prop == 'preview': - return db.PropertyMetadata(blob=_prop_path(guid, prop), - mime_type='image/png') - elif prop == 'data': - return db.PropertyMetadata(blob=_ds_path(guid, 'data'), - mime_type=get(guid, 'mime_type') or 'application/octet') - else: - response.content_type = 'application/json' - return get(guid, prop) - def _ds_path(guid, *args): return os.path.join(_ds_root, guid[:2], guid, *args) diff --git a/sugar_network/client/commands.py b/sugar_network/client/routes.py index d20b2aa..709bba2 100644 --- a/sugar_network/client/commands.py +++ b/sugar_network/client/routes.py @@ -18,14 +18,14 @@ import logging import httplib from os.path import join -from sugar_network import db, client, node, toolkit -from sugar_network.toolkit import netlink, mountpoints +from sugar_network import db, client, node, toolkit, model from sugar_network.client import journal, clones, injector -from sugar_network.resources.volume import Volume, Commands -from sugar_network.node.slave import SlaveCommands +from sugar_network.node.slave import SlaveRoutes +from sugar_network.toolkit import netlink, mountpoints +from sugar_network.toolkit.router import ACL, Request, Response, Router +from sugar_network.toolkit.router import route, fallbackroute from sugar_network.toolkit.spec import Spec -from sugar_network.toolkit import zeroconf, coroutine, util, http -from sugar_network.toolkit import exception, enforce +from sugar_network.toolkit import zeroconf, coroutine, http, enforce # Top-level directory name to keep SN data on mounted devices @@ -38,19 +38,17 @@ _SYNC_DIRNAME = 'sugar-network-sync' _RECONNECT_TIMEOUT = 3 _RECONNECT_TIMEOUT_MAX = 60 * 15 -_logger = logging.getLogger('client.commands') +_logger = logging.getLogger('client.routes') -class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): +class ClientRoutes(model.Routes, journal.Routes): - def __init__(self, home_volume, api_url=None, no_subscription=False, - static_prefix=None): - db.CommandsProcessor.__init__(self) - Commands.__init__(self) + def __init__(self, home_volume, api_url=None, no_subscription=False): + model.Routes.__init__(self) if not client.no_dbus.value: - journal.Commands.__init__(self) + journal.Routes.__init__(self) - self._home = _VolumeCommands(home_volume) + self._local = _LocalRoutes(home_volume) self._inline = coroutine.Event() self._inline_job = coroutine.Pool() self._remote_urls = [] @@ -59,15 +57,7 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): self._no_subscription = no_subscription self._server_mode = not api_url - self._accept_language = client.accept_language.value - if not self._accept_language: - self._accept_language = [toolkit.default_lang()] - - if not static_prefix: - static_prefix = 'http://127.0.0.1:%s' % client.ipc_port.value - self._static_prefix = static_prefix - - home_volume.connect(self.broadcast) + home_volume.broadcast = self.broadcast if self._server_mode: mountpoints.connect(_SN_DIRNAME, @@ -82,9 +72,9 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): def close(self): self._jobs.kill() self._got_offline() - self._home.volume.close() + self._local.volume.close() - @db.route('GET', '/hub') + @route('GET', ['hub']) def hub(self, request, response): """Serve Hub via HTTP instead of file:// for IPC users. @@ -111,7 +101,7 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): return file(path, 'rb') - @db.route('GET', '/packages') + @fallbackroute('GET', ['packages']) def route_packages(self, request, response): if self._inline.is_set(): return self._node_call(request, response) @@ -120,7 +110,7 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): # no way to process specified request on the node raise http.ServiceUnavailable() - @db.volume_command(method='GET', cmd='status', + @route('GET', cmd='status', mime_type='application/json') def status(self): result = {'route': 'proxy' if self._inline.is_set() else 'offline'} @@ -128,60 +118,50 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): result['node'] = self._node.api_url return result - @db.volume_command(method='GET', cmd='inline', + @route('GET', cmd='inline', mime_type='application/json') def inline(self): if not self._server_mode and not self._inline.is_set(): self._remote_connect() return self._inline.is_set() - @db.volume_command(method='GET', cmd='whoami', - mime_type='application/json') def whoami(self, request, response): - try: - result = self._node_call(request, response) - except db.CommandNotFound: - result = {'roles': [], 'guid': client.sugar_uid()} - return result + if self._inline.is_set(): + return self._node_call(request, response) + else: + return {'roles': [], 'guid': client.sugar_uid()} - @db.directory_command(method='GET', - arguments={ - 'reply': db.to_list, - 'clone': db.to_int, - 'favorite': db.to_bool, - }, + @route('GET', [None], + arguments={'reply': ('guid',), 'clone': int, 'favorite': bool}, mime_type='application/json') - def find(self, request, response, document, reply, clone, favorite): + def find(self, request, response, clone, favorite): if not self._inline.is_set() or clone or favorite: - return self._home.call(request, response) + return self._local.call(request, response) else: return self._proxy_get(request, response) - @db.document_command(method='GET', - arguments={'reply': db.to_list}, mime_type='application/json') + @route('GET', [None, None], + arguments={'reply': list}, mime_type='application/json') def get(self, request, response): return self._proxy_get(request, response) - @db.property_command(method='GET', mime_type='application/json') - def get_prop(self, request, response, document, guid): + @route('GET', [None, None, None], mime_type='application/json') + def get_prop(self, request, response): return self._proxy_get(request, response) - @db.document_command(method='GET', cmd='make') - def make(self, document, guid): - enforce(document == 'context', 'Only contexts can be launched') - - for event in injector.make(guid): + @route('GET', ['context', None], cmd='make') + def make(self, request): + for event in injector.make(request.guid): event['event'] = 'make' self.broadcast(event) - @db.document_command(method='GET', cmd='launch', - arguments={'args': db.to_list}) - def launch(self, document, guid, args, activity_id=None, + @route('GET', ['context', None], cmd='launch', + arguments={'args': list}) + def launch(self, request, args, activity_id=None, object_id=None, uri=None, color=None, no_spawn=None): - enforce(document == 'context', 'Only contexts can be launched') def do_launch(): - for event in injector.launch(guid, args, + for event in injector.launch(request.guid, args, activity_id=activity_id, object_id=object_id, uri=uri, color=color): event['event'] = 'launch' @@ -192,76 +172,71 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): else: self._jobs.spawn(do_launch) - @db.document_command(method='PUT', cmd='clone', - arguments={ - 'force': db.to_int, - 'nodeps': db.to_int, - 'requires': db.to_list, - }) - def clone(self, request, document, guid, force): + @route('PUT', ['context', None], cmd='clone', + arguments={'force': False, 'nodeps': False, 'requires': list}) + def clone_context(self, request): enforce(self._inline.is_set(), 'Not available in offline') - if document == 'context': - context_type = self._node_call(method='GET', document='context', - guid=guid, prop='type') - if 'activity' in context_type: - self._clone_activity(guid, request) - elif 'content' in context_type: - - def get_props(): - impls = self._node_call(method='GET', - document='implementation', context=guid, - stability='stable', order_by='-version', limit=1, - reply=['guid'])['result'] - enforce(impls, http.NotFound, 'No implementations') - impl_id = impls[0]['guid'] - props = self._node_call(method='GET', document='context', - guid=guid, reply=['title', 'description']) - props['preview'] = self._node_call(method='GET', - document='context', guid=guid, prop='preview') - data_response = db.Response() - props['data'] = self._node_call(response=data_response, - method='GET', document='implementation', - guid=impl_id, prop='data') - props['mime_type'] = data_response.content_type or \ - 'application/octet' - props['activity_id'] = impl_id - return props - - self._clone_jobject(guid, request.content, get_props, force) - else: - raise RuntimeError('No way to clone') - elif document == 'artifact': + context_type = self._node_call(method='GET', + path=['context', request.guid, 'type']) + + if 'activity' in context_type: + self._clone_activity(request) + elif 'content' in context_type: def get_props(): - props = self._node_call(method='GET', document='artifact', - guid=guid, reply=['title', 'description', 'context']) + impls = self._node_call(method='GET', + path=['implementation'], context=request.guid, + stability='stable', order_by='-version', limit=1, + reply=['guid'])['result'] + enforce(impls, http.NotFound, 'No implementations') + impl_id = impls[0]['guid'] + props = self._node_call(method='GET', + path=['context', request.guid], + reply=['title', 'description']) props['preview'] = self._node_call(method='GET', - document='artifact', guid=guid, prop='preview') - props['data'] = self._node_call(method='GET', - document='artifact', guid=guid, prop='data') - props['activity'] = props.pop('context') + path=['context', request.guid, 'preview']) + data_response = Response() + props['data'] = self._node_call(response=data_response, + method='GET', + path=['implementation', impl_id, 'data']) + props['mime_type'] = data_response.content_type or \ + 'application/octet' + props['activity_id'] = impl_id return props - self._clone_jobject(guid, request.content, get_props, force) + self._clone_jobject(request, get_props) else: - raise RuntimeError('Command is not supported for %r' % document) + raise RuntimeError('No way to clone') - @db.document_command(method='PUT', cmd='favorite') - def favorite(self, request, document, guid): - if document == 'context': - if request.content or self._home.volume['context'].exists(guid): - self._checkin_context(guid, {'favorite': request.content}) - else: - raise RuntimeError('Command is not supported for %r' % document) + @route('PUT', ['artifact', None], cmd='clone', arguments={'force': False}) + def clone_artifact(self, request): + enforce(self._inline.is_set(), 'Not available in offline') - @db.document_command(method='GET', cmd='feed', + def get_props(): + props = self._node_call(method='GET', + path=['artifact', request.guid], + reply=['title', 'description', 'context']) + props['preview'] = self._node_call(method='GET', + path=['artifact', request.guid, 'preview']) + props['data'] = self._node_call(method='GET', + path=['artifact', request.guid, 'data']) + props['activity'] = props.pop('context') + return props + + self._clone_jobject(request, get_props) + + @route('PUT', ['context', None], cmd='favorite') + def favorite(self, request): + if request.content or \ + self._local.volume['context'].exists(request.guid): + self._checkin_context(request.guid, {'favorite': request.content}) + + @route('GET', ['context', None], cmd='feed', mime_type='application/json') - def feed(self, document, guid, layer, distro, request, response): - enforce(document == 'context') - + def feed(self, request, response): try: - context = self._home.volume['context'].get(guid) + context = self._local.volume['context'].get(request.guid) except http.NotFound: context = None if context is None or context['clone'] != 2: @@ -277,7 +252,7 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): try: spec = Spec(root=path) except Exception: - exception(_logger, 'Failed to read %r spec file', path) + toolkit.exception(_logger, 'Failed to read %r spec file', path) continue versions.append({ 'guid': spec.root, @@ -297,23 +272,14 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): 'implementations': versions, } - def call(self, request, response=None): - request.static_prefix = self._static_prefix - request.accept_language = self._accept_language - request.allow_redirects = True - try: - return db.CommandsProcessor.call(self, request, response) - except db.CommandNotFound: - return self._node_call(request, response) - - def _node_call(self, request=None, response=None, **kwargs): + @fallbackroute() + def _node_call(self, request=None, response=None, method=None, path=None, + **kwargs): if request is None: - request = db.Request(**kwargs) - request.static_prefix = self._static_prefix - request.accept_language = self._accept_language - request.allow_redirects = True + request = Request(method=method, path=path) + request.update(kwargs) if self._inline.is_set(): - if client.layers.value and request.get('document') in \ + if client.layers.value and request.resource in \ ('context', 'implementation') and \ 'layer' not in request: request['layer'] = client.layers.value @@ -325,9 +291,9 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): return reply except (http.ConnectionError, httplib.IncompleteRead): self._restart_online() - return self._home.call(request, response) + return self._local.call(request, response) else: - return self._home.call(request, response) + return self._local.call(request, response) def _got_online(self): enforce(not self._inline.is_set()) @@ -359,16 +325,16 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): self._remote_connect() def _wait_for_connectivity(self): - for route in netlink.wait_for_route(): + for i in netlink.wait_for_route(): self._fall_offline() - if route: + if i: self._remote_connect() def _remote_connect(self, timeout=0): def pull_events(): for event in self._node.subscribe(): - if event.get('document') == 'implementation': + if event.get('resource') == 'implementation': mtime = event.get('mtime') if mtime: injector.invalidate_solutions(mtime) @@ -402,7 +368,8 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): _logger.debug('Retry %r on gateway error', url) continue except Exception: - exception(_logger, 'Connection to %r failed', url) + toolkit.exception(_logger, + 'Connection to %r failed', url) break self._got_offline() if not timeout: @@ -428,14 +395,13 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): node.stats_root.value = join(root, _SN_DIRNAME, 'stats') node.files_root.value = join(root, _SN_DIRNAME, 'files') - volume = Volume(db_path) - self._node = _PersonalCommands(join(db_path, 'node'), volume, + volume = db.Volume(db_path, model.RESOURCES) + self._node = _NodeRoutes(join(db_path, 'node'), volume, self.broadcast) self._jobs.spawn(volume.populate) logging.info('Start %r node on %s port', volume.root, node.port.value) - server = coroutine.WSGIServer(('0.0.0.0', node.port.value), - db.Router(self._node)) + server = coroutine.WSGIServer(('0.0.0.0', node.port.value), self._node) self._inline_job.spawn(server.serve_forever) self._got_online() @@ -447,22 +413,13 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): self._inline_job.kill() self._got_offline() - def _clone_jobject(self, uid, value, get_props, force): - if value: - if force or not journal.exists(uid): - self.journal_update(uid, **get_props()) - self.broadcast({'event': 'show_journal', 'uid': uid}) - else: - if journal.exists(uid): - self.journal_delete(uid) - def _checkin_context(self, guid, props): - contexts = self._home.volume['context'] + contexts = self._local.volume['context'] if contexts.exists(guid): contexts.update(guid, props) else: - copy = self._node_call(method='GET', document='context', guid=guid, + copy = self._node_call(method='GET', path=['context', guid], reply=[ 'type', 'title', 'summary', 'description', 'homepage', 'mime_types', 'dependencies', @@ -471,62 +428,33 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): copy['guid'] = guid contexts.create(copy) for prop in ('icon', 'artifact_icon', 'preview'): - blob = self._node_call(method='GET', document='context', - guid=guid, prop=prop) + blob = self._node_call(method='GET', + path=['context', guid, prop]) if blob is not None: contexts.update(guid, {prop: {'blob': blob}}) - def _clone_activity(self, guid, request): - if not request.content: - clones.wipeout(guid) - return - - for __ in clones.walk(guid): - if not request.get('force'): - return - break - - self._checkin_context(guid, {'clone': 1}) - - if request.get('nodeps'): - pipe = injector.clone_impl(guid, - stability=request.get('stability'), - requires=request.get('requires')) - else: - pipe = injector.clone(guid) - - for event in pipe: - event['event'] = 'clone' - self.broadcast(event) - - for __ in clones.walk(guid): - break - else: - # Cloning was failed - self._checkin_context(guid, {'clone': 0}) - def _proxy_get(self, request, response): - document = request['document'] - if document not in ('context', 'artifact'): + resource = request.resource + if resource not in ('context', 'artifact'): return self._node_call(request, response) if not self._inline.is_set(): - return self._home.call(request, response) + return self._local.call(request, response) - request_guid = request.get('guid') - if request_guid and self._home.volume[document].exists(request_guid): - return self._home.call(request, response) + request_guid = request.guid if len(request.path) > 1 else None + if request_guid and self._local.volume[resource].exists(request_guid): + return self._local.call(request, response) - if 'prop' in request: + if request.prop is not None: mixin = None else: reply = request.setdefault('reply', ['guid']) mixin = set(reply) & _LOCAL_PROPS if mixin: # Otherwise there is no way to mixin _LOCAL_PROPS - if 'guid' not in request and 'guid' not in reply: + if not request_guid and 'guid' not in reply: reply.append('guid') - if document == 'context' and 'type' not in reply: + if resource == 'context' and 'type' not in reply: reply.append('type') result = self._node_call(request, response) @@ -544,8 +472,8 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): if 'favorite' in mixin: props['favorite'] = bool(int(journal.get(guid, 'keep') or 0)) - if document == 'context': - contexts = self._home.volume['context'] + if resource == 'context': + contexts = self._local.volume['context'] for props in items: guid = request_guid or props['guid'] if 'activity' in props['type']: @@ -557,45 +485,80 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): props.update(patch) elif 'content' in props['type']: mixin_jobject(props, guid) - elif document == 'artifact': + elif resource == 'artifact': for props in items: mixin_jobject(props, request_guid or props['guid']) return result + def _clone_activity(self, request): + if not request.content: + clones.wipeout(request.guid) + return + for __ in clones.walk(request.guid): + if not request.get('force'): + return + break + self._checkin_context(request.guid, {'clone': 1}) + if request.get('nodeps'): + pipe = injector.clone_impl(request.guid, + stability=request.get('stability'), + requires=request.get('requires')) + else: + pipe = injector.clone(request.guid) + for event in pipe: + event['event'] = 'clone' + self.broadcast(event) + for __ in clones.walk(request.guid): + break + else: + # Cloning was failed + self._checkin_context(request.guid, {'clone': 0}) + + def _clone_jobject(self, request, get_props): + if request.content: + if request['force'] or not journal.exists(request.guid): + self.journal_update(request.guid, **get_props()) + self.broadcast({ + 'event': 'show_journal', + 'uid': request.guid, + }) + else: + if journal.exists(request.guid): + self.journal_delete(request.guid) + -class CachedClientCommands(ClientCommands): +class CachedClientRoutes(ClientRoutes): - def __init__(self, home_volume, api_url=None, no_subscription=False, - static_prefix=None): - ClientCommands.__init__(self, home_volume, api_url, no_subscription, - static_prefix) - self._push_seq = util.PersistentSequence( + def __init__(self, home_volume, api_url=None, no_subscription=False): + ClientRoutes.__init__(self, home_volume, api_url, no_subscription) + self._push_seq = toolkit.PersistentSequence( join(home_volume.root, 'push.sequence'), [1, None]) self._push_job = coroutine.Pool() def _got_online(self): - ClientCommands._got_online(self) + ClientRoutes._got_online(self) self._push_job.spawn(self._push) def _got_offline(self): self._push_job.kill() - ClientCommands._got_offline(self) + ClientRoutes._got_offline(self) def _push(self): - pushed_seq = util.Sequence() - skiped_seq = util.Sequence() + pushed_seq = toolkit.Sequence() + skiped_seq = toolkit.Sequence() def push(request, seq): try: self._node.call(request) except Exception: - exception(_logger, 'Cannot push %r, will postpone', request) + toolkit.exception(_logger, + 'Cannot push %r, will postpone', request) skiped_seq.include(seq) else: pushed_seq.include(seq) - for document, directory in self._home.volume.items(): + for document, directory in self._local.volume.items(): if directory.mtime <= self._push_seq.mtime: continue @@ -603,19 +566,20 @@ class CachedClientCommands(ClientCommands): for guid, patch in directory.diff(self._push_seq, layer='local'): diff = {} - diff_seq = util.Sequence() + diff_seq = toolkit.Sequence() post_requests = [] for prop, meta, seqno in patch: if 'blob' in meta: - request = db.Request(method='PUT', document=document, - guid=guid, prop=prop) + request = Request(method='PUT', + path=[document, guid, prop]) request.content_type = meta['mime_type'] request.content_length = os.stat(meta['blob']).st_size - request.content_stream = util.iter_file(meta['blob']) + request.content_stream = \ + toolkit.iter_file(meta['blob']) post_requests.append((request, seqno)) elif 'url' in meta: - request = db.Request(method='PUT', document=document, - guid=guid, prop=prop) + request = Request(method='PUT', + path=[document, guid, prop]) request.content_type = 'application/json' request.content = meta post_requests.append((request, seqno)) @@ -624,16 +588,14 @@ class CachedClientCommands(ClientCommands): diff_seq.include(seqno, seqno) if not diff: continue - request = db.Request(document=document) if 'guid' in diff: - request['method'] = 'POST' - access = db.ACCESS_CREATE | db.ACCESS_WRITE + request = Request(method='POST', path=[document]) + access = ACL.CREATE | ACL.WRITE else: - request['method'] = 'PUT' - request['guid'] = guid - access = db.ACCESS_WRITE + request = Request(method='PUT', path=[document, guid]) + access = ACL.WRITE for name in diff.keys(): - if not (directory.metadata[name].permissions & access): + if not (directory.metadata[name].acl & access): del diff[name] request.content_type = 'application/json' request.content = diff @@ -653,29 +615,31 @@ class CachedClientCommands(ClientCommands): # No any decent reasons to keep fail reports after uploding. # TODO The entire offlile synchronization should be improved, # for now, it is possible to have a race here - self._home.volume['report'].wipe() + self._local.volume['report'].wipe() self._push_seq.commit() self.broadcast({'event': 'push'}) -class _VolumeCommands(db.VolumeCommands): +class _LocalRoutes(db.Routes, Router): def __init__(self, volume): - db.VolumeCommands.__init__(self, volume) + db.Routes.__init__(self, volume) + Router.__init__(self, self) def on_create(self, request, props, event): props['layer'] = tuple(props['layer']) + ('local',) - db.VolumeCommands.on_create(self, request, props, event) + db.Routes.on_create(self, request, props, event) -class _PersonalCommands(SlaveCommands): +class _NodeRoutes(SlaveRoutes, Router): def __init__(self, key_path, volume, localcast): - SlaveCommands.__init__(self, key_path, volume) + SlaveRoutes.__init__(self, key_path, volume) + Router.__init__(self, self) self.api_url = 'http://127.0.0.1:%s' % node.port.value self._localcast = localcast - self._mounts = util.Pool() + self._mounts = toolkit.Pool() self._jobs = coroutine.Pool() users = volume['user'] @@ -686,22 +650,18 @@ class _PersonalCommands(SlaveCommands): mountpoints.connect(_SYNC_DIRNAME, self.__found_mountcb, self.__lost_mount_cb) - volume.connect(localcast) - @db.volume_command(method='GET', cmd='whoami', - mime_type='application/json') - def whoami(self, request): - return {'roles': [], 'guid': client.sugar_uid()} + def preroute(self, op, request): + request.principal = client.sugar_uid() - def validate(self, *args): - return True + def whoami(self, request, response): + return {'roles': [], 'guid': client.sugar_uid()} - def call(self, request, response=None): - request.principal = client.sugar_uid() - return SlaveCommands.call(self, request, response) + def broadcast(self, event=None, request=None): + SlaveRoutes.broadcast(self, event, request) + self._localcast(event) def close(self): - self.volume.disconnect(self._localcast) self.volume.close() def __repr__(self): @@ -718,7 +678,8 @@ class _PersonalCommands(SlaveCommands): join(mountpoint, _SYNC_DIRNAME), **(self._offline_session or {})) except Exception, error: - exception(_logger, 'Failed to complete synchronization') + toolkit.exception(_logger, + 'Failed to complete synchronization') self._localcast({'event': 'sync_abort', 'error': str(error)}) self._offline_session = None raise @@ -740,7 +701,7 @@ class _PersonalCommands(SlaveCommands): self._jobs.spawn(self._sync_mounts) def __lost_mount_cb(self, path): - if self._mounts.remove(path) == util.Pool.ACTIVE: + if self._mounts.remove(path) == toolkit.Pool.ACTIVE: _logger.warning('%r was unmounted, break synchronization', path) self._jobs.kill() diff --git a/sugar_network/db/__init__.py b/sugar_network/db/__init__.py index c39ac2c..0a77a4d 100644 --- a/sugar_network/db/__init__.py +++ b/sugar_network/db/__init__.py @@ -349,29 +349,12 @@ Volume """ -from sugar_network.db.env import \ - ACCESS_CREATE, ACCESS_WRITE, ACCESS_READ, ACCESS_DELETE, \ - ACCESS_AUTHOR, ACCESS_AUTH, ACCESS_PUBLIC, ACCESS_LEVELS, \ - ACCESS_SYSTEM, ACCESS_LOCAL, ACCESS_REMOTE, ACCESS_CALC, \ - MAX_LIMIT, CommandNotFound, gettext, \ - index_flush_timeout, index_flush_threshold, index_write_queue - -from sugar_network.db.router import route, Router - from sugar_network.db.metadata import \ indexed_property, stored_property, blob_property, \ - Property, StoredProperty, BlobProperty, IndexedProperty, \ - PropertyMetadata - -from sugar_network.db.commands import \ - volume_command, volume_command_pre, volume_command_post, \ - directory_command, directory_command_pre, directory_command_post, \ - document_command, document_command_pre, document_command_post, \ - property_command, property_command_pre, property_command_post, \ - to_int, to_list, to_bool, Request, Response, CommandsProcessor - -from sugar_network.db.document import Document - -from sugar_network.db.directory import Directory - -from sugar_network.db.volume import Volume, VolumeCommands + Property, StoredProperty, BlobProperty, IndexedProperty +from sugar_network.db.index import index_flush_timeout, \ + index_flush_threshold, index_write_queue +from sugar_network.db.resource import Resource +from sugar_network.db.directory import Directory, MAX_LIMIT +from sugar_network.db.volume import Volume +from sugar_network.db.routes import Routes diff --git a/sugar_network/db/commands.py b/sugar_network/db/commands.py deleted file mode 100644 index c6ff71a..0000000 --- a/sugar_network/db/commands.py +++ /dev/null @@ -1,458 +0,0 @@ -# Copyright (C) 2012 Aleksey Lim -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -import logging -from email.utils import formatdate - -from sugar_network import toolkit -from sugar_network.db import env -from sugar_network.db.metadata import PropertyMetadata -from sugar_network.toolkit import http, enforce - - -_logger = logging.getLogger('db.commands') - - -def db_command(scope, **kwargs): - - def decorate(func): - func.scope = scope - func.kwargs = kwargs - return func - - return decorate - - -volume_command = \ - lambda ** kwargs: db_command('volume', **kwargs) -volume_command_pre = \ - lambda ** kwargs: db_command('volume', wrapper='pre', **kwargs) -volume_command_post = \ - lambda ** kwargs: db_command('volume', wrapper='post', **kwargs) - -directory_command = \ - lambda ** kwargs: db_command('directory', **kwargs) -directory_command_pre = \ - lambda ** kwargs: db_command('directory', wrapper='pre', **kwargs) -directory_command_post = \ - lambda ** kwargs: db_command('directory', wrapper='post', **kwargs) - -document_command = \ - lambda ** kwargs: db_command('document', **kwargs) -document_command_pre = \ - lambda ** kwargs: db_command('document', wrapper='pre', **kwargs) -document_command_post = \ - lambda ** kwargs: db_command('document', wrapper='post', **kwargs) - -property_command = \ - lambda ** kwargs: db_command('property', **kwargs) -property_command_pre = \ - lambda ** kwargs: db_command('property', wrapper='pre', **kwargs) -property_command_post = \ - lambda ** kwargs: db_command('property', wrapper='post', **kwargs) - - -def to_int(value): - if isinstance(value, basestring): - if not value: - return 0 - enforce(value.isdigit(), 'Argument should be an integer value') - return int(value) - - -def to_bool(value): - if isinstance(value, basestring): - return int(value.strip().lower() in ('true', '1', 'on')) - return int(bool(value)) - - -def to_list(value): - if isinstance(value, basestring): - if value: - return value.split(',') - else: - return [] - return value - - -class Request(dict): - - #: Request payload, e.g., content passed by a HTTP POST/PUT request - content = None - #: If payload is a stream, :attr:`content` will be ``None`` in that case - content_stream = None - #: Payload stream length, if :attr:`content_stream` is set - content_length = None - #: Payload MIME type - content_type = None - access_level = env.ACCESS_REMOTE - accept_language = None - commands = None - response = None - static_prefix = None - principal = None - if_modified_since = None - allow_redirects = False - path = None - - def __init__(self, **kwargs): - """Initialize parameters dictionary using named arguments.""" - dict.__init__(self, kwargs) - self._pos = 0 - - def __getitem__(self, key): - enforce(key in self, 'Cannot find %r request argument', key) - return self.get(key) - - def read(self, size=None): - if self.content_stream is None: - return '' - rest = max(0, self.content_length - self._pos) - size = rest if size is None else min(rest, size) - result = self.content_stream.read(size) - if not result: - return '' - self._pos += len(result) - return result - - def clone(self): - request = type(self)() - request.access_level = self.access_level - request.accept_language = self.accept_language - request.commands = self.commands - return request - - def call(self, method, content=None, content_stream=None, - content_length=None, **kwargs): - enforce(self.commands is not None) - - request = self.clone() - request.update(kwargs) - request['method'] = method - request.content = content - request.content_stream = content_stream - request.content_length = content_length - - return self.commands.call(request, Response()) - - def __repr__(self): - args = ['content_length=%r' % self.content_length, - 'access_level=%r' % self.access_level, - 'accept_language=%r' % self.accept_language, - ] + ['%s=%r' % i for i in self.items()] - return '<db.Request %s>' % ' '.join(args) - - -class Response(dict): - - def __init__(self, **kwargs): - """Initialize parameters dictionary using named arguments.""" - dict.__init__(self, kwargs) - self.meta = {} - - @property - def content_length(self): - return int(self.get('content-length') or '0') - - @content_length.setter - def content_length(self, value): - self.set('content-length', value) - - @property - def content_type(self): - return self.get('content-type') - - @content_type.setter - def content_type(self, value): - if value: - self.set('content-type', value) - elif 'content-type' in self: - self.remove('content-type') - - @property - def last_modified(self): - return self.get('last-modified') - - @last_modified.setter - def last_modified(self, value): - self.set('last-modified', - formatdate(value, localtime=False, usegmt=True)) - - def items(self): - result = [] - for key, value in dict.items(self): - if type(value) in (list, tuple): - for i in value: - result.append((key, str(i))) - else: - result.append((key, str(value))) - return result - - def __repr__(self): - args = ['%s=%r' % i for i in self.items()] - return '<Response %s>' % ' '.join(args) - - def __contains__(self, key): - dict.__contains__(self, key.lower()) - - def __getitem__(self, key): - return self.get(key.lower()) - - def __setitem__(self, key, value): - return self.set(key.lower(), value) - - def __delitem__(self, key, value): - self.remove(key.lower()) - - def set(self, key, value): - dict.__setitem__(self, key, value) - - def remove(self, key): - dict.__delitem__(self, key) - - -class CommandsProcessor(object): - - def __init__(self, volume=None): - self._routes = {} - self._commands = { - 'volume': _Commands(), - 'directory': _Commands(), - 'document': _Commands(), - 'property': _Commands(), - } - self.volume = volume - - self._scan_for_routes() - - for scope, kwargs in _scan_class(self.__class__, False): - cmd = _Command((self,), **kwargs) - self._commands[scope].add(cmd) - - if volume is not None: - for directory in volume.values(): - for scope, kwargs in _scan_class(directory.document_class): - cmd = _ObjectCommand(directory, **kwargs) - self._commands[scope].add(cmd) - - def super_call(self, request, response): - """Will be called if no commands were recognized. - - This function needs to be overloaded in child classes to implement - proxy commands processor. - - """ - raise env.CommandNotFound() - - def call(self, request, response=None): - """Make a command call. - - :param request: - :class:`Request` object with call parameters - :param response: - optional :class:`Response` object to collect response details - :returns: - command call result - - """ - if request.path is not None: - rout = self._routes.get(( - request['method'], - request.path[0] if request.path else '')) - if rout: - return rout(self, request, response) - - cmd = self.resolve(request) - enforce(cmd is not None, env.CommandNotFound, 'Unsupported command') - - enforce(request.access_level & cmd.access_level, http.Forbidden, - 'Operation is permitted on requester\'s level') - - if response is None: - response = Response() - request.commands = self - request.response = response - - if not request.accept_language: - request.accept_language = [toolkit.default_lang()] - - for arg, cast in cmd.arguments.items(): - if arg not in request: - continue - try: - request[arg] = cast(request[arg]) - except Exception, error: - raise RuntimeError('Cannot typecast %r command argument: %s' % - (arg, error)) - - args = cmd.get_args(request) - - for pre in cmd.pre: - pre(*args, request=request) - - kwargs = {} - for arg in cmd.kwarg_names: - if arg == 'request': - kwargs[arg] = request - elif arg == 'response': - kwargs[arg] = response - elif arg not in kwargs: - kwargs[arg] = request.get(arg) - - result = cmd.callback(*args, **kwargs) - - for post in cmd.post: - result = post(*args, result=result, request=request, - response=response) - - if not response.content_type: - if isinstance(result, PropertyMetadata): - response.content_type = result.get('mime_type') - if not response.content_type: - response.content_type = cmd.mime_type - - return result - - def resolve(self, request): - """Recognize particular command from a :class:`Request` object. - - :param request: - request object to recognize command from, the process is based - on ``method`` and ``cmd`` parameters - :returns: - command object or ``None`` - - """ - key = (request.get('method', 'GET'), request.get('cmd'), None) - - if 'document' not in request: - return self._commands['volume'].get(key) - - document_key = key[:2] + (request['document'],) - - if 'guid' not in request: - commands = self._commands['directory'] - return commands.get(key) or commands.get(document_key) - - if 'prop' not in request: - commands = self._commands['document'] - return commands.get(key) or commands.get(document_key) - - commands = self._commands['property'] - return commands.get(key) or commands.get(document_key) - - def _scan_for_routes(self): - cls = self.__class__ - while cls is not None: - for name in dir(cls): - attr = getattr(cls, name) - if hasattr(attr, 'route'): - self._routes[attr.route] = attr - # pylint: disable-msg=E1101 - cls = cls.__base__ - - -class _Command(object): - - def __init__(self, args, callback, method='GET', document=None, cmd=None, - mime_type=None, permissions=0, access_level=env.ACCESS_LEVELS, - arguments=None, pre=None, post=None): - self.args = args - self.callback = callback - self.mime_type = mime_type - self.permissions = permissions - self.access_level = access_level - self.kwarg_names = _function_arg_names(callback) - self.key = (method, cmd, document) - self.arguments = arguments or {} - self.pre = pre - self.post = post - - def get_args(self, request): - return self.args - - def __repr__(self): - return '%s(method=%s, cmd=%s, document=%s)' % \ - ((self.callback.__name__,) + self.key) - - -class _ObjectCommand(_Command): - - def __init__(self, directory, **kwargs): - _Command.__init__(self, (), document=directory.metadata.name, **kwargs) - self._directory = directory - - def get_args(self, request): - document = self._directory.get(request['guid']) - document.request = request - return (document,) - - -class _Commands(dict): - - def add(self, cmd): - enforce(cmd.key not in self, 'Command %r already exists', cmd) - self[cmd.key] = cmd - - -def _function_arg_names(func): - if hasattr(func, 'im_func'): - func = func.im_func - if not hasattr(func, 'func_code'): - return [] - code = func.func_code - # `1:` is for skipping the first, `self` or `cls`, argument - return code.co_varnames[1:code.co_argcount] - - -def _scan_class(root_cls, is_document_class=True): - processed = set() - commands = {} - - cls = root_cls - while cls is not None: - for name in dir(cls): - if name in processed: - continue - attr = getattr(cls, name) - if not hasattr(attr, 'scope'): - continue - enforce(not is_document_class or - attr.scope in ('document', 'property'), - 'Wrong scale command') - key = (attr.scope, - attr.kwargs.get('method') or 'GET', - attr.kwargs.get('cmd')) - kwargs = commands.setdefault(key, {'pre': [], 'post': []}) - callback = getattr(root_cls, attr.__name__) - if 'wrapper' not in attr.kwargs: - kwargs.update(attr.kwargs) - kwargs['callback'] = callback - else: - for key in ('arguments',): - if key in attr.kwargs and key not in kwargs: - kwargs[key] = attr.kwargs[key] - kwargs[attr.kwargs['wrapper']].append(callback) - processed.add(name) - cls = cls.__base__ - - for (scope, method, cmd), kwargs in commands.items(): - if 'callback' not in kwargs: - kwargs['method'] = method - if cmd: - kwargs['cmd'] = cmd - kwargs['callback'] = lambda self, request, response: \ - self.super_call(request, response) - yield scope, kwargs diff --git a/sugar_network/db/directory.py b/sugar_network/db/directory.py index 52f12ef..08f1c35 100644 --- a/sugar_network/db/directory.py +++ b/sugar_network/db/directory.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2012 Aleksey Lim +# Copyright (C) 2011-2013 Aleksey Lim # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,13 +19,15 @@ import logging from os.path import exists, join from sugar_network import toolkit -from sugar_network.db import env +from sugar_network.toolkit.router import ACL from sugar_network.db.storage import Storage from sugar_network.db.metadata import BlobProperty, Metadata, GUID_PREFIX from sugar_network.db.metadata import IndexedProperty, StoredProperty -from sugar_network.toolkit import http, util, exception, enforce +from sugar_network.toolkit import http, exception, enforce +MAX_LIMIT = 2147483648 + # To invalidate existed index on stcuture changes _LAYOUT_VERSION = 4 @@ -35,7 +37,7 @@ _logger = logging.getLogger('db.directory') class Directory(object): def __init__(self, root, document_class, index_class, - notification_cb=None, seqno=None): + broadcast=None, seqno=None): """ :param index_class: what class to use to access to indexes, for regular casses @@ -50,14 +52,13 @@ class Directory(object): # Metadata cannot be recreated document_class.metadata = Metadata(document_class) document_class.metadata['guid'] = IndexedProperty('guid', - slot=0, prefix=GUID_PREFIX, - permissions=env.ACCESS_CREATE | env.ACCESS_READ) + slot=0, prefix=GUID_PREFIX, acl=ACL.CREATE | ACL.READ) self.metadata = document_class.metadata self.document_class = document_class + self.broadcast = broadcast or (lambda event: None) self._index_class = index_class self._root = root - self._notification_cb = notification_cb self._seqno = _SessionSeqno() if seqno is None else seqno self._storage = None self._index = None @@ -71,7 +72,7 @@ class Directory(object): @mtime.setter def mtime(self, value): self._index.mtime = value - self._notify({'event': 'populate', 'mtime': value}) + self.broadcast({'event': 'populate', 'mtime': value}) def wipe(self): self.close() @@ -239,7 +240,7 @@ class Directory(object): self._index.checkpoint() self._save_layout() self.commit() - self._notify({'event': 'populate', 'mtime': self.mtime}) + self.broadcast({'event': 'populate', 'mtime': self.mtime}) def diff(self, seq, exclude_seq=None, **params): if exclude_seq is None: @@ -250,7 +251,7 @@ class Directory(object): else: params['order_by'] = 'seqno' # TODO On big requests, xapian can raise an exception on edits - params['limit'] = env.MAX_LIMIT + params['limit'] = MAX_LIMIT params['no_cache'] = True for start, end in seq: @@ -263,8 +264,7 @@ class Directory(object): def patch(): for name, prop in self.metadata.items(): - if name == 'seqno' or \ - prop.permissions & env.ACCESS_CALC: + if name == 'seqno' or prop.acl & ACL.CALC: continue meta = doc.meta(name) if meta is None: @@ -364,24 +364,19 @@ class Directory(object): def _post_store(self, guid, changes, event=None): if event is not None: - self._notify(event) + self.broadcast(event) def _post_delete(self, guid, event): self._storage.delete(guid) - self._notify(event) + self.broadcast(event) def _post_commit(self): self._seqno.commit() - self._notify({'event': 'commit', 'mtime': self.mtime}) - - def _notify(self, event): - if self._notification_cb is not None: - event['document'] = self.metadata.name - self._notification_cb(event) + self.broadcast({'event': 'commit', 'mtime': self.mtime}) def _save_layout(self): path = join(self._root, 'layout') - with util.new_file(path) as f: + with toolkit.new_file(path) as f: f.write(str(_LAYOUT_VERSION)) def _is_layout_stale(self): diff --git a/sugar_network/db/env.py b/sugar_network/db/env.py deleted file mode 100644 index 09ed43f..0000000 --- a/sugar_network/db/env.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright (C) 2011-2013 Aleksey Lim -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -from sugar_network import toolkit -from sugar_network.toolkit import Option, http - - -ACCESS_CREATE = 1 -ACCESS_WRITE = 2 -ACCESS_READ = 4 -ACCESS_DELETE = 8 -ACCESS_PUBLIC = ACCESS_CREATE | ACCESS_WRITE | ACCESS_READ | ACCESS_DELETE - -ACCESS_AUTH = 16 -ACCESS_AUTHOR = 32 - -ACCESS_SYSTEM = 64 -ACCESS_LOCAL = 128 -ACCESS_REMOTE = 256 -ACCESS_LEVELS = ACCESS_SYSTEM | ACCESS_LOCAL | ACCESS_REMOTE - -ACCESS_CALC = 512 - -ACCESS_NAMES = { - ACCESS_CREATE: 'Create', - ACCESS_WRITE: 'Write', - ACCESS_READ: 'Read', - ACCESS_DELETE: 'Delete', - } - -MAX_LIMIT = 2147483648 - - -index_flush_timeout = Option( - 'flush index index after specified seconds since the last change', - default=5, type_cast=int) - -index_flush_threshold = Option( - 'flush index every specified changes', - default=32, type_cast=int) - -index_write_queue = Option( - 'if active-document is being used for the scheme with one writer ' - 'process and multiple reader processes, this option specifies ' - 'the writer\'s queue size', - default=256, type_cast=int) - - -def gettext(value, accept_language=None): - if not value: - return '' - if not isinstance(value, dict): - return value - - if accept_language is None: - accept_language = [toolkit.default_lang()] - elif isinstance(accept_language, basestring): - accept_language = [accept_language] - accept_language.append('en') - - stripped_value = None - for lang in accept_language: - result = value.get(lang) - if result is not None: - return result - - prime_lang = lang.split('-')[0] - if prime_lang != lang: - result = value.get(prime_lang) - if result is not None: - return result - - if stripped_value is None: - stripped_value = {} - for k, v in value.items(): - if '-' in k: - stripped_value[k.split('-', 1)[0]] = v - result = stripped_value.get(prime_lang) - if result is not None: - return result - - return value[min(value.keys())] - - -class CommandNotFound(http.BadRequest): - pass diff --git a/sugar_network/db/index.py b/sugar_network/db/index.py index cd456ed..3c8579f 100644 --- a/sugar_network/db/index.py +++ b/sugar_network/db/index.py @@ -22,9 +22,24 @@ from os.path import exists, join import xapian -from sugar_network.db import env -from sugar_network.db.metadata import IndexedProperty, GUID_PREFIX -from sugar_network.toolkit import coroutine, exception, enforce +from sugar_network import toolkit +from sugar_network.db.metadata import IndexedProperty, GUID_PREFIX, LIST_TYPES +from sugar_network.toolkit import Option, coroutine, exception, enforce + + +index_flush_timeout = Option( + 'flush index index after specified seconds since the last change', + default=5, type_cast=int) + +index_flush_threshold = Option( + 'flush index every specified changes', + default=32, type_cast=int) + +index_write_queue = Option( + 'if active-document is being used for the scheme with one writer ' + 'process and multiple reader processes, this option specifies ' + 'the writer\'s queue size', + default=256, type_cast=int) # Additional Xapian term prefix for exact search terms _EXACT_PREFIX = 'X' @@ -113,7 +128,7 @@ class IndexReader(object): def find(self, query): """Search documents within the index. - Function interface is the same as for `db.Document.find`. + Function interface is the same as for `db.Resource.find`. """ start_timestamp = time.time() @@ -181,7 +196,9 @@ class IndexReader(object): for needle in value if type(value) in (tuple, list) else [value]: if needle is None: continue - needle = prop.to_string(needle)[0] + if prop.parse is not None: + needle = prop.parse(needle) + needle = next(_fmt_prop_value(prop, needle)) if needle.startswith('!'): term = _term(prop.prefix, needle[1:]) not_queries.append(xapian.Query(term)) @@ -334,13 +351,13 @@ class IndexWriter(IndexReader): value_ = xapian.sortable_serialise(value) else: if prop.localized: - value_ = env.gettext(value) or '' + value_ = toolkit.gettext(value) or '' else: - value_ = prop.to_string(value)[0] + value_ = next(_fmt_prop_value(prop, value)) document.add_value(prop.slot, value_) if prop.prefix or prop.full_text: - for value_ in prop.to_string(value): + for value_ in _fmt_prop_value(prop, value): if prop.prefix: if prop.boolean: document.add_boolean_term( @@ -418,14 +435,14 @@ class IndexWriter(IndexReader): self._commit_cb() def _check_for_commit(self): - if env.index_flush_threshold.value > 0 and \ - self._pending_updates >= env.index_flush_threshold.value: + if index_flush_threshold.value > 0 and \ + self._pending_updates >= index_flush_threshold.value: # Avoid processing heavy commits in the same coroutine self._commit_cond.set() def _commit_handler(self): - if env.index_flush_timeout.value > 0: - timeout = env.index_flush_timeout.value + if index_flush_timeout.value > 0: + timeout = index_flush_timeout.value else: timeout = None @@ -437,3 +454,20 @@ class IndexWriter(IndexReader): def _term(prefix, value): return _EXACT_PREFIX + prefix + str(value).split('\n')[0][:243] + + +def _fmt_prop_value(prop, value): + + def fmt(value): + if type(value) is unicode: + yield value.encode('utf8') + elif isinstance(value, basestring): + yield value + elif type(value) in LIST_TYPES: + for i in value: + for j in fmt(i): + yield j + else: + yield str(value) + + return fmt(value if prop.fmt is None else prop.fmt(value)) diff --git a/sugar_network/db/metadata.py b/sugar_network/db/metadata.py index ea797a3..d22bb6f 100644 --- a/sugar_network/db/metadata.py +++ b/sugar_network/db/metadata.py @@ -13,20 +13,17 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import os import types -import json -from os.path import exists from sugar_network import toolkit -from sugar_network.db import env +from sugar_network.toolkit.router import ACL from sugar_network.toolkit import http, enforce #: Xapian term prefix for GUID value GUID_PREFIX = 'I' -_LIST_TYPES = (list, tuple, frozenset, types.GeneratorType) +LIST_TYPES = (list, tuple, frozenset, types.GeneratorType) def indexed_property(property_class=None, *args, **kwargs): @@ -71,7 +68,7 @@ class Metadata(dict): def __init__(self, cls): """ :param cls: - class inherited from `db.Document` + class inherited from `db.Resource` """ self._name = cls.__name__.lower() @@ -106,7 +103,7 @@ class Metadata(dict): @property def name(self): - """Document type name.""" + """Resource type name.""" return self._name def __getitem__(self, prop_name): @@ -115,34 +112,24 @@ class Metadata(dict): return dict.__getitem__(self, prop_name) -class PropertyMetadata(dict): - - BLOB_SUFFIX = '.blob' - - def __init__(self, path_=None, **meta): - if path_: - with file(path_) as f: - meta.update(json.load(f)) - blob_path = path_ + PropertyMetadata.BLOB_SUFFIX - if exists(blob_path): - meta['blob'] = blob_path - meta['blob_size'] = os.stat(blob_path).st_size - meta['mtime'] = int(os.stat(path_).st_mtime) - dict.__init__(self, meta) - - class Property(object): """Basic class to collect information about document property.""" - def __init__(self, name, permissions=env.ACCESS_PUBLIC, typecast=None, - reprcast=None, default=None): + def __init__(self, name, acl=ACL.PUBLIC, typecast=None, + parse=None, fmt=None, default=None): + if typecast is bool: + if fmt is None: + fmt = lambda x: '1' if x else '0' + if parse is None: + parse = lambda x: str(x).lower() in ('true', '1', 'on', 'yes') self.setter = None self.on_get = lambda self, x: x self.on_set = None self._name = name - self._permissions = permissions + self._acl = acl self._typecast = typecast - self._reprcast = reprcast + self._parse = parse + self._fmt = fmt self._default = default @property @@ -151,14 +138,14 @@ class Property(object): return self._name @property - def permissions(self): + def acl(self): """Specify access to the property. Value might be ORed composition of `db.ACCESS_*` constants. """ - return self._permissions + return self._acl @property def typecast(self): @@ -177,40 +164,20 @@ class Property(object): return self._typecast @property - def composite(self): - """Is property value a list of values.""" - is_composite, __ = _is_composite(self.typecast) - return is_composite + def parse(self): + """Parse property value from a string.""" + return self._parse + + @property + def fmt(self): + """Format property value to a string or a list of strings.""" + return self._fmt @property def default(self): """Default property value or None.""" return self._default - def decode(self, value): - """Convert property value according to its `typecast`.""" - if self.typecast is None: - return value - return _decode(self.typecast, value) - - def to_string(self, value): - """Convert value to list of strings ready to index.""" - result = [] - - if self._reprcast is not None: - value = self._reprcast(value) - - for subvalue in (value if type(value) in _LIST_TYPES else [value]): - if type(subvalue) is bool: - subvalue = int(subvalue) - if type(subvalue) is unicode: - subvalue = unicode(subvalue).encode('utf8') - else: - subvalue = str(subvalue) - result.append(subvalue) - - return result - def assert_access(self, mode): """Is access to the property permitted. @@ -222,15 +189,15 @@ class Property(object): to specify the access mode """ - enforce(mode & self.permissions, http.Forbidden, + enforce(mode & self.acl, http.Forbidden, '%s access is disabled for %r property', - env.ACCESS_NAMES[mode], self.name) + ACL.NAMES[mode], self.name) class StoredProperty(Property): """Property to save only in persistent storage, no index.""" - def __init__(self, name, localized=False, typecast=None, reprcast=None, + def __init__(self, name, localized=False, typecast=None, fmt=None, **kwargs): """ :param: **kwargs @@ -242,13 +209,12 @@ class StoredProperty(Property): if localized: enforce(typecast is None, 'typecast should be None for localized properties') - enforce(reprcast is None, - 'reprcast should be None for localized properties') + enforce(fmt is None, + 'fmt should be None for localized properties') typecast = _localized_typecast - reprcast = _localized_reprcast + fmt = _localized_fmt - Property.__init__(self, name, typecast=typecast, reprcast=reprcast, - **kwargs) + Property.__init__(self, name, typecast=typecast, fmt=fmt, **kwargs) @property def localized(self): @@ -310,16 +276,15 @@ class IndexedProperty(StoredProperty): class BlobProperty(Property): """Binary large objects which needs to be fetched alone, no index.""" - def __init__(self, name, permissions=env.ACCESS_PUBLIC, - mime_type='application/octet-stream', composite=False): + def __init__(self, name, acl=ACL.PUBLIC, + mime_type='application/octet-stream'): """ :param: **kwargs :class:`.Property` arguments """ - Property.__init__(self, name, permissions=permissions) + Property.__init__(self, name, acl=acl) self._mime_type = mime_type - self._composite = composite @property def mime_type(self): @@ -330,63 +295,11 @@ class BlobProperty(Property): """ return self._mime_type - @property - def composite(self): - """Property is a list of BLOBs.""" - return self._composite - - -def _is_composite(typecast): - if type(typecast) in _LIST_TYPES: - if typecast: - first = iter(typecast).next() - if type(first) is not type and \ - type(first) not in _LIST_TYPES: - return False, True - return True, False - return False, False - - -def _decode(typecast, value): - enforce(value is not None, ValueError, 'Property value cannot be None') - - is_composite, is_enum = _is_composite(typecast) - - if is_composite: - enforce(len(typecast) <= 1, ValueError, - 'List values should contain values of the same type') - if type(value) not in _LIST_TYPES: - value = (value,) - typecast, = typecast or [str] - value = tuple([_decode(typecast, i) for i in value]) - elif is_enum: - enforce(value in typecast, ValueError, - "Value %r is not in '%s' list", - value, ', '.join([str(i) for i in typecast])) - elif isinstance(typecast, types.FunctionType): - value = typecast(value) - elif typecast is str: - if isinstance(value, unicode): - value = value.encode('utf-8') - else: - value = str(value) - elif typecast is int: - value = int(value) - elif typecast is float: - value = float(value) - elif typecast is bool: - value = bool(value) - elif typecast is dict: - value = dict(value) - else: - raise ValueError('Unknown typecast') - return value - def _is_sloted_prop(typecast): if typecast in [None, int, float, bool, str]: return True - if type(typecast) in _LIST_TYPES: + if type(typecast) in LIST_TYPES: if typecast and [i for i in typecast if type(i) in [None, int, float, bool, str]]: return True @@ -399,7 +312,7 @@ def _localized_typecast(value): return {toolkit.default_lang(): value} -def _localized_reprcast(value): +def _localized_fmt(value): if isinstance(value, dict): return value.values() else: diff --git a/sugar_network/db/document.py b/sugar_network/db/resource.py index bdc9660..7484a89 100644 --- a/sugar_network/db/document.py +++ b/sugar_network/db/resource.py @@ -13,17 +13,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import logging +from sugar_network import toolkit +from sugar_network.db.metadata import StoredProperty, indexed_property +from sugar_network.toolkit.router import Blob, ACL -from sugar_network.db import env -from sugar_network.db.metadata import StoredProperty, PropertyMetadata -from sugar_network.db.metadata import indexed_property - -_logger = logging.getLogger('db.document') - - -class Document(object): +class Resource(object): """Base class for all data classes.""" #: `Metadata` object that describes the document @@ -39,27 +34,53 @@ class Document(object): @property def volume(self): - return self.request.commands.volume + return self.request.routes.volume @property def directory(self): return self.volume[self.metadata.name] - @indexed_property(slot=1000, prefix='IC', typecast=int, - permissions=env.ACCESS_READ, default=0) + @indexed_property(slot=1000, prefix='RC', typecast=int, default=0, + acl=ACL.READ) def ctime(self, value): return value - @indexed_property(slot=1001, prefix='IM', typecast=int, - permissions=env.ACCESS_READ, default=0) + @indexed_property(slot=1001, prefix='RM', typecast=int, default=0, + acl=ACL.READ) def mtime(self, value): return value - @indexed_property(slot=1002, prefix='IS', typecast=int, - permissions=0, default=0) + @indexed_property(slot=1002, prefix='RS', typecast=int, default=0, acl=0) def seqno(self, value): return value + @indexed_property(prefix='RA', typecast=dict, full_text=True, default={}, + fmt=lambda x: _fmt_authors(x), acl=ACL.READ) + def author(self, value): + result = [] + for guid, props in sorted(value.items(), + cmp=lambda x, y: cmp(x[1]['order'], y[1]['order'])): + if 'name' in props: + result.append({ + 'guid': guid, + 'name': props['name'], + 'role': props['role'], + }) + else: + result.append({ + 'name': guid, + 'role': props['role'], + }) + return result + + @indexed_property(prefix='RL', typecast=[], default=['public']) + def layer(self, value): + return value + + @indexed_property(prefix='RT', full_text=True, default=[], typecast=[]) + def tags(self, value): + return value + def get(self, prop, accept_language=None): """Get document's property value. @@ -80,12 +101,12 @@ class Document(object): else: value = prop.default else: - value = meta or PropertyMetadata() + value = meta or Blob() self.props[prop.name] = value if value is not None and accept_language: if isinstance(prop, StoredProperty) and prop.localized: - value = env.gettext(value, accept_language) + value = toolkit.gettext(value, accept_language) return value @@ -107,3 +128,17 @@ class Document(object): def __setitem__(self, prop, value): self.props[prop] = value self._modifies.add(prop) + + +def _fmt_authors(value): + if isinstance(value, dict): + for guid, props in value.items(): + if not isinstance(props, dict): + yield guid + else: + if 'name' in props: + yield props['name'] + if not (props['role'] & ACL.INSYSTEM): + yield guid + else: + yield value diff --git a/sugar_network/db/router.py b/sugar_network/db/router.py deleted file mode 100644 index f7f4b96..0000000 --- a/sugar_network/db/router.py +++ /dev/null @@ -1,381 +0,0 @@ -# Copyright (C) 2012-2013 Aleksey Lim -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -import os -import cgi -import json -import time -import types -import logging -import mimetypes -from email.utils import parsedate -from urlparse import parse_qsl, urlsplit -from bisect import bisect_left -from os.path import join, isfile, split, splitext - -from sugar_network import static -from sugar_network.db import env -from sugar_network.db.commands import Request, Response -from sugar_network.db.metadata import PropertyMetadata -from sugar_network.toolkit import BUFFER_SIZE -from sugar_network.toolkit import http, coroutine, exception, enforce - - -_logger = logging.getLogger('router') - - -def route(method, path): - path = path.strip('/').split('/') - # Only top level paths for now - enforce(len(path) == 1) - - def decorate(func): - func.route = (method, path[0]) - return func - - return decorate - - -class Router(object): - - def __init__(self, commands): - self.commands = commands - self._authenticated = set() - self._valid_origins = set() - self._invalid_origins = set() - self._host = None - - if 'SSH_ASKPASS' in os.environ: - # Otherwise ssh-keygen will popup auth dialogs on registeration - del os.environ['SSH_ASKPASS'] - - def authenticate(self, request): - user = request.environ.get('HTTP_SUGAR_USER') - if user is None: - return None - - if user not in self._authenticated and \ - (request.path != ['user'] or request['method'] != 'POST'): - _logger.debug('Logging %r user', user) - request = Request(method='GET', cmd='exists', - document='user', guid=user) - enforce(self.commands.call(request), http.Unauthorized, - 'Principal does not exist') - self._authenticated.add(user) - - return user - - def call(self, request, response): - if 'HTTP_ORIGIN' in request.environ: - enforce(self._assert_origin(request.environ), http.Forbidden, - 'Cross-site is not allowed for %r origin', - request.environ['HTTP_ORIGIN']) - response['Access-Control-Allow-Origin'] = \ - request.environ['HTTP_ORIGIN'] - - if request['method'] == 'OPTIONS': - # TODO Process OPTIONS request per url? - if request.environ['HTTP_ORIGIN']: - response['Access-Control-Allow-Methods'] = \ - request.environ['HTTP_ACCESS_CONTROL_REQUEST_METHOD'] - response['Access-Control-Allow-Headers'] = \ - request.environ['HTTP_ACCESS_CONTROL_REQUEST_HEADERS'] - else: - response['Allow'] = 'GET, POST, PUT, DELETE' - response.content_length = 0 - return None - - request.principal = self.authenticate(request) - if request.path[:1] == ['static']: - path = join(static.PATH, *request.path[1:]) - result = PropertyMetadata(blob=path, - mime_type=_get_mime_type(path), filename=split(path)[-1]) - else: - result = self.commands.call(request, response) - - if isinstance(result, PropertyMetadata): - if 'url' in result: - raise http.Redirect(result['url']) - - path = result['blob'] - enforce(isfile(path), 'No such file') - - mtime = result.get('mtime') or os.stat(path).st_mtime - if request.if_modified_since and mtime and \ - mtime <= request.if_modified_since: - raise http.NotModified() - response.last_modified = mtime - - response.content_type = result.get('mime_type') or \ - 'application/octet-stream' - - filename = result.get('filename') - if not filename: - filename = _filename(result.get('name') or - splitext(split(path)[-1])[0], - response.content_type) - response['Content-Disposition'] = \ - 'attachment; filename="%s"' % filename - - result = file(path, 'rb') - - if hasattr(result, 'read'): - if hasattr(result, 'fileno'): - response.content_length = os.fstat(result.fileno()).st_size - elif hasattr(result, 'seek'): - result.seek(0, 2) - response.content_length = result.tell() - result.seek(0) - result = _stream_reader(result) - - return result - - def __call__(self, environ, start_response): - request = _Request(environ) - request_repr = str(request) if _logger.level <= logging.DEBUG else None - response = _Response() - - js_callback = None - if 'callback' in request: - js_callback = request.pop('callback') - - result = None - try: - result = self.call(request, response) - except http.StatusPass, error: - response.status = error.status - if error.headers: - response.update(error.headers) - response.content_type = None - except Exception, error: - exception('Error while processing %r request', request.url) - if isinstance(error, http.Status): - response.status = error.status - response.update(error.headers or {}) - result = error.result - else: - response.status = '500 Internal Server Error' - if result is None: - result = {'error': str(error), - 'request': request.url, - } - response.content_type = 'application/json' - - result_streamed = isinstance(result, types.GeneratorType) - - if request['method'] == 'HEAD': - result_streamed = False - result = None - elif js_callback: - if result_streamed: - result = ''.join(result) - result_streamed = False - result = '%s(%s);' % (js_callback, json.dumps(result)) - response.content_length = len(result) - elif not result_streamed: - if response.content_type == 'application/json': - result = json.dumps(result) - if 'content-length' not in response: - response.content_length = len(result) if result else 0 - - for key, value in response.meta.items(): - response.set('X-SN-%s' % str(key), json.dumps(value)) - - _logger.trace('Called %s: response=%r result=%r streamed=%r', - request_repr, response, result, result_streamed) - - start_response(response.status, response.items()) - - if request['method'] == 'HEAD': - enforce(result is None, 'HEAD responses should not contain body') - elif result_streamed: - for i in result: - yield i - elif result is not None: - yield result - - def _assert_origin(self, environ): - origin = environ['HTTP_ORIGIN'] - if origin in self._valid_origins: - return True - if origin in self._invalid_origins: - return False - - valid = True - if origin == 'null' or origin.startswith('file://'): - # True all time for local apps - pass - else: - if self._host is None: - http_host = environ['HTTP_HOST'].split(':', 1)[0] - self._host = coroutine.gethostbyname(http_host) - ip = coroutine.gethostbyname(urlsplit(origin).hostname) - valid = (self._host == ip) - - if valid: - _logger.info('Allow cross-site for %r origin', origin) - self._valid_origins.add(origin) - else: - _logger.info('Disallow cross-site for %r origin', origin) - self._invalid_origins.add(origin) - return valid - - -class _Request(Request): - - environ = None - url = None - - def __init__(self, environ=None): - Request.__init__(self) - - if not environ: - return - - http_host = environ.get('HTTP_HOST') - if http_host: - self.static_prefix = 'http://' + http_host - self.access_level = env.ACCESS_REMOTE - self.environ = environ - self.url = '/' + environ['PATH_INFO'].strip('/') - self.path = [i for i in self.url[1:].split('/') if i] - self['method'] = environ['REQUEST_METHOD'] - self.content = None - self.content_stream = environ.get('wsgi.input') - self.content_length = 0 - self.accept_language = _parse_accept_language( - environ.get('HTTP_ACCEPT_LANGUAGE')) - self.principal = None - self.query = {} - - enforce('..' not in self.path, 'Relative url path') - - query = environ.get('QUERY_STRING') or '' - for attr, value in parse_qsl(query): - attr = str(attr) - param = self.get(attr) - if type(param) is list: - param.append(value) - else: - if param is not None: - value = [param, value] - self[attr] = value - if attr != 'cmd': - self.query[attr] = value - if query: - self.url += '?' + query - - content_length = environ.get('CONTENT_LENGTH') - if content_length: - self.content_length = int(content_length) - self.content_type, __ = \ - cgi.parse_header(environ.get('CONTENT_TYPE', '')) - if self.content_type.lower() == 'application/json': - self.content = json.load(self.content_stream) - elif self.content_type.lower() == 'multipart/form-data': - files = cgi.FieldStorage(fp=environ['wsgi.input'], - environ=environ) - enforce(len(files.list) == 1, - 'Multipart request should contain only one file') - self.content_stream = files.list[0].file - - if_modified_since = environ.get('HTTP_IF_MODIFIED_SINCE') - if if_modified_since: - if_modified_since = parsedate(if_modified_since) - enforce(if_modified_since is not None, - 'Failed to parse If-Modified-Since') - self.if_modified_since = time.mktime(if_modified_since) - - scope = len(self.path) - if scope == 3: - self['document'], self['guid'], self['prop'] = self.path - elif scope == 2: - self['document'], self['guid'] = self.path - elif scope == 1: - self['document'], = self.path - - def clone(self): - request = Request.clone(self) - request.environ = self.environ - request.url = self.url - request.path = self.path - request.principal = self.principal - return request - - -class _Response(Response): - - status = '200 OK' - - -def _parse_accept_language(accept_language): - if not accept_language: - return [] - - langs = [] - qualities = [] - - for chunk in accept_language.split(','): - lang, params = (chunk.split(';', 1) + [None])[:2] - lang = lang.strip() - if not lang: - continue - - quality = 1 - if params: - params = params.split('=', 1) - if len(params) > 1 and params[0].strip() == 'q': - quality = float(params[1]) - - index = bisect_left(qualities, quality) - qualities.insert(index, quality) - langs.insert(len(langs) - index, lang.lower().replace('_', '-')) - - return langs - - -def _get_mime_type(path): - if not mimetypes.inited: - mimetypes.init() - suffix = '.' + path.rsplit('.', 1)[-1] - return mimetypes.types_map.get(suffix) - - -def _filename(names, mime_type): - if type(names) not in (list, tuple): - names = [names] - parts = [] - for name in names: - if isinstance(name, dict): - name = env.gettext(name) - parts.append(''.join([i.capitalize() for i in str(name).split()])) - result = '-'.join(parts) - if mime_type: - if not mimetypes.inited: - mimetypes.init() - result += mimetypes.guess_extension(mime_type) or '' - return result.replace(os.sep, '') - - -def _stream_reader(stream): - try: - while True: - chunk = stream.read(BUFFER_SIZE) - if not chunk: - break - yield chunk - finally: - if hasattr(stream, 'close'): - stream.close() diff --git a/sugar_network/db/routes.py b/sugar_network/db/routes.py new file mode 100644 index 0000000..e61199a --- /dev/null +++ b/sugar_network/db/routes.py @@ -0,0 +1,381 @@ +# Copyright (C) 2011-2013 Aleksey Lim +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import re +import sys +import time +import json +import types +import hashlib +import logging +from contextlib import contextmanager +from os.path import exists + +from sugar_network import toolkit +from sugar_network.db.metadata import BlobProperty, StoredProperty, LIST_TYPES +from sugar_network.toolkit.router import Blob, ACL, route +from sugar_network.toolkit import http, enforce + + +_GUID_RE = re.compile('[a-zA-Z0-9_+-.]+$') + +_logger = logging.getLogger('db.routes') + + +class Routes(object): + + def __init__(self, volume): + self.volume = volume + + @route('POST', [None], + acl=ACL.AUTH, mime_type='application/json') + def create(self, request): + with self._post(request, ACL.CREATE) as (directory, doc): + event = {} + self.on_create(request, doc.props, event) + if 'guid' not in doc.props: + doc.props['guid'] = toolkit.uuid() + doc.guid = doc.props['guid'] + directory.create(doc.props, event) + return doc.guid + + @route('GET', [None], + arguments={'offset': int, 'limit': int, 'reply': ('guid',)}, + mime_type='application/json') + def find(self, request, reply): + self._preget(request) + documents, total = self.volume[request.resource].find(**request) + result = [self._get_props(i, request, reply) for i in documents] + return {'total': total, 'result': result} + + @route('GET', [None, None], cmd='exists', + mime_type='application/json') + def exists(self, request): + directory = self.volume[request.resource] + return directory.exists(request.guid) + + @route('PUT', [None, None], + acl=ACL.AUTH | ACL.AUTHOR) + def update(self, request): + with self._post(request, ACL.WRITE) as (directory, doc): + if not doc.props: + return + event = {} + self.on_update(request, doc.props, event) + directory.update(doc.guid, doc.props, event) + + @route('PUT', [None, None, None], + acl=ACL.AUTH | ACL.AUTHOR) + def update_prop(self, request, url=None): + if url: + value = Blob({'url': url}) + elif request.content is None: + value = request.content_stream + else: + value = request.content + request.content = {request.prop: value} + self.update(request) + + @route('DELETE', [None, None], + acl=ACL.AUTH | ACL.AUTHOR) + def delete(self, request): + self.volume[request.resource].delete(request.guid) + + @route('GET', [None, None], arguments={'reply': ('guid',)}, + mime_type='application/json') + def get(self, request, reply): + if not reply: + reply = [] + for prop in self.volume[request.resource].metadata.values(): + if prop.acl & ACL.READ and not (prop.acl & ACL.LOCAL): + reply.append(prop.name) + self._preget(request) + doc = self.volume[request.resource].get(request.guid) + return self._get_props(doc, request, reply) + + @route('GET', [None, None, None], mime_type='application/json') + def get_prop(self, request): + directory = self.volume[request.resource] + prop = directory.metadata[request.prop] + doc = directory.get(request.guid) + doc.request = request + + prop.assert_access(ACL.READ) + + if isinstance(prop, StoredProperty): + value = doc.get(prop.name, request.accept_language) + value = prop.on_get(doc, value) + if value is None: + value = prop.default + return value + else: + meta = prop.on_get(doc, doc.meta(prop.name)) + enforce(meta is not None and ('blob' in meta or 'url' in meta), + http.NotFound, 'BLOB does not exist') + return meta + + @route('HEAD', [None, None, None]) + def get_prop_meta(self, request, response): + directory = self.volume[request.resource] + prop = directory.metadata[request.prop] + doc = directory.get(request.guid) + doc.request = request + + prop.assert_access(ACL.READ) + + if isinstance(prop, StoredProperty): + meta = doc.meta(prop.name) + value = meta.pop('value') + response.content_length = len(json.dumps(value)) + else: + meta = prop.on_get(doc, doc.meta(prop.name)) + enforce(meta is not None and ('blob' in meta or 'url' in meta), + http.NotFound, 'BLOB does not exist') + if 'blob' in meta: + meta.pop('blob') + meta['url'] = '/'.join([request.static_prefix] + request.path) + response.content_length = meta['blob_size'] + + response.meta.update(meta) + response.last_modified = meta['mtime'] + + @route('PUT', [None, None], cmd='useradd', + arguments={'role': 0}, acl=ACL.AUTH | ACL.AUTHOR) + def useradd(self, request, user, role): + enforce(user, "Argument 'user' is not specified") + directory = self.volume[request.resource] + authors = directory.get(request.guid)['author'] + self._useradd(authors, user, role) + directory.update(request.guid, {'author': authors}) + + @route('PUT', [None, None], cmd='userdel', acl=ACL.AUTH | ACL.AUTHOR) + def userdel(self, request, user): + enforce(user, "Argument 'user' is not specified") + enforce(user != request.principal, 'Cannot remove yourself') + directory = self.volume[request.resource] + authors = directory.get(request.guid)['author'] + enforce(user in authors, 'No such user') + del authors[user] + directory.update(request.guid, {'author': authors}) + + def on_create(self, request, props, event): + if 'guid' in props: + # TODO Temporal security hole, see TODO + guid = props['guid'] + enforce(not self.volume[request.resource].exists(guid), + '%s already exists', guid) + enforce(_GUID_RE.match(guid) is not None, + 'Malformed %s GUID', guid) + + ts = int(time.time()) + props['ctime'] = ts + props['mtime'] = ts + + if request.principal: + authors = props['author'] = {} + self._useradd(authors, request.principal, ACL.ORIGINAL) + + def on_update(self, request, props, event): + props['mtime'] = int(time.time()) + + def after_post(self, doc): + pass + + @contextmanager + def _post(self, request, access): + content = request.content or {} + enforce(isinstance(content, dict), 'Invalid value') + + directory = self.volume[request.resource] + if request.guid: + doc = directory.get(request.guid) + else: + doc = directory.document_class(None, {}) + doc.request = request + blobs = [] + + for name, value in content.items(): + prop = directory.metadata[name] + if isinstance(prop, BlobProperty): + prop.assert_access(ACL.CREATE if + access == ACL.WRITE and doc.meta(name) is None + else access) + if value is None: + value = {'blob': None} + elif isinstance(value, dict): + enforce('url' in value, + 'Key %r is not specified in %r blob property', + 'url', name) + value = {'url': value['url']} + else: + value = _read_blob(request, prop, value) + blobs.append(value['blob']) + else: + prop.assert_access(access) + if prop.localized and isinstance(value, basestring): + value = {request.accept_language[0]: value} + try: + value = _typecast_prop_value(prop.typecast, value) + except Exception, error: + error = 'Value %r for %r property is invalid: %s' % \ + (value, prop.name, error) + toolkit.exception(error) + raise RuntimeError(error) + doc[name] = value + + if access == ACL.CREATE: + for name, prop in directory.metadata.items(): + if not isinstance(prop, BlobProperty) and \ + name not in content and \ + (prop.default is not None or prop.on_set is not None): + doc[name] = prop.default + + try: + for name, value in doc.props.items(): + prop = directory.metadata[name] + if prop.on_set is not None: + doc.props[name] = prop.on_set(doc, value) + yield directory, doc + finally: + for path in blobs: + if exists(path): + os.unlink(path) + + self.after_post(doc) + + def _preget(self, request): + reply = request['reply'] + if not reply: + request['reply'] = ('guid',) + else: + directory = self.volume[request.resource] + for prop in reply: + directory.metadata[prop].assert_access(ACL.READ) + + def _get_props(self, doc, request, props): + result = {} + metadata = doc.metadata + doc.request = request + for name in props: + prop = metadata[name] + value = prop.on_get(doc, doc.get(name, request.accept_language)) + if value is None: + value = prop.default + elif isinstance(value, Blob): + value = value.get('url') + if value is None: + value = '/'.join(['', metadata.name, doc.guid, name]) + if value.startswith('/'): + value = request.static_prefix + value + result[name] = value + return result + + def _useradd(self, authors, user, role): + props = {} + + users = self.volume['user'] + if users.exists(user): + props['name'] = users.get(user)['name'] + role |= ACL.INSYSTEM + else: + role &= ~ACL.INSYSTEM + props['role'] = role & (ACL.INSYSTEM | ACL.ORIGINAL) + + if user in authors: + authors[user].update(props) + else: + if authors: + top = max(authors.values(), key=lambda x: x['order']) + props['order'] = top['order'] + 1 + else: + props['order'] = 0 + authors[user] = props + + +def _read_blob(request, prop, value): + digest = hashlib.sha1() + dst = toolkit.NamedTemporaryFile(delete=False) + + try: + if isinstance(value, basestring): + digest.update(value) + dst.write(value) + else: + size = request.content_length or sys.maxint + while size > 0: + chunk = value.read(min(size, toolkit.BUFFER_SIZE)) + if not chunk: + break + dst.write(chunk) + size -= len(chunk) + digest.update(chunk) + except Exception: + os.unlink(dst.name) + raise + finally: + dst.close() + + return {'blob': dst.name, + 'digest': digest.hexdigest(), + 'mime_type': request.content_type or prop.mime_type, + } + + +def _typecast_prop_value(typecast, value): + if typecast is None: + return value + enforce(value is not None, ValueError, 'Property value cannot be None') + + def cast(typecast, value): + if isinstance(typecast, types.FunctionType): + return typecast(value) + elif typecast is unicode: + return value.encode('utf-8') + elif typecast is str: + return str(value) + elif typecast is int: + return int(value) + elif typecast is float: + return float(value) + elif typecast is bool: + return bool(value) + elif typecast is dict: + return dict(value) + else: + raise ValueError('Unknown typecast') + + if type(typecast) in LIST_TYPES: + if typecast: + first = iter(typecast).next() + else: + first = None + if first is not None and type(first) is not type and \ + type(first) not in LIST_TYPES: + value = cast(type(first), value) + enforce(value in typecast, ValueError, + "Value %r is not in '%s' list", + value, ', '.join([str(i) for i in typecast])) + else: + enforce(len(typecast) <= 1, ValueError, + 'List values should contain values of the same type') + if type(value) not in LIST_TYPES: + value = (value,) + typecast, = typecast or [str] + value = tuple([_typecast_prop_value(typecast, i) for i in value]) + else: + value = cast(typecast, value) + + return value diff --git a/sugar_network/db/storage.py b/sugar_network/db/storage.py index 8416dee..69d8896 100644 --- a/sugar_network/db/storage.py +++ b/sugar_network/db/storage.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Aleksey Lim +# Copyright (C) 2012-2013 Aleksey Lim # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -20,8 +20,12 @@ import shutil import cPickle as pickle from os.path import exists, join, isdir, basename -from sugar_network.db.metadata import PropertyMetadata, BlobProperty -from sugar_network.toolkit import util, exception +from sugar_network import toolkit +from sugar_network.toolkit.router import Blob +from sugar_network.db.metadata import BlobProperty + + +_BLOB_SUFFIX = '.blob' class Storage(object): @@ -55,7 +59,7 @@ class Storage(object): try: shutil.rmtree(path) except Exception, error: - exception() + toolkit.exception() raise RuntimeError('Cannot delete %r document from %r: %s' % (guid, self.metadata.name, error)) @@ -126,8 +130,16 @@ class Record(object): def get(self, prop): path = join(self._root, prop) - if exists(path): - return PropertyMetadata(path) + if not exists(path): + return None + with file(path) as f: + meta = Blob(json.load(f)) + blob_path = path + _BLOB_SUFFIX + if exists(blob_path): + meta['blob'] = blob_path + meta['blob_size'] = os.stat(blob_path).st_size + meta['mtime'] = int(os.stat(path).st_mtime) + return meta def set(self, prop, mtime=None, **meta): if not exists(self._root): @@ -135,17 +147,17 @@ class Record(object): meta_path = join(self._root, prop) if 'blob' in meta: - dst_blob_path = meta_path + PropertyMetadata.BLOB_SUFFIX + dst_blob_path = meta_path + _BLOB_SUFFIX blob = meta.pop('blob') if hasattr(blob, 'read'): - with util.new_file(dst_blob_path) as f: + with toolkit.new_file(dst_blob_path) as f: shutil.copyfileobj(blob, f) elif blob is not None: os.rename(blob, dst_blob_path) elif exists(dst_blob_path): os.unlink(dst_blob_path) - with util.new_file(meta_path) as f: + with toolkit.new_file(meta_path) as f: json.dump(meta, f) if mtime: os.utime(meta_path, (mtime, mtime)) diff --git a/sugar_network/db/volume.py b/sugar_network/db/volume.py index 18e0b2c..03af0fc 100644 --- a/sugar_network/db/volume.py +++ b/sugar_network/db/volume.py @@ -14,30 +14,15 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import os -import re -import sys -import time -import json -import hashlib import logging -from contextlib import contextmanager from os.path import exists, join, abspath from sugar_network import toolkit -from sugar_network.db import env from sugar_network.db.directory import Directory from sugar_network.db.index import IndexWriter -from sugar_network.db.commands import CommandsProcessor, directory_command -from sugar_network.db.commands import document_command, property_command -from sugar_network.db.commands import to_int, to_list -from sugar_network.db.metadata import BlobProperty, StoredProperty -from sugar_network.db.metadata import PropertyMetadata -from sugar_network.toolkit import http, coroutine, util -from sugar_network.toolkit import BUFFER_SIZE, exception, enforce +from sugar_network.toolkit import http, coroutine, enforce -_GUID_RE = re.compile('[a-zA-Z0-9_+-.]+$') - _logger = logging.getLogger('db.volume') @@ -45,8 +30,10 @@ class Volume(dict): _flush_pool = [] - def __init__(self, root, documents, index_class=None): + def __init__(self, root, documents, broadcast=None, index_class=None): Volume._flush_pool.append(self) + self.broadcast = broadcast or (lambda event: None) + self._populators = coroutine.Pool() if index_class is None: index_class = IndexWriter @@ -57,8 +44,7 @@ class Volume(dict): if not exists(root): os.makedirs(root) self._index_class = index_class - self._subscriptions = {} - self.seqno = util.Seqno(join(self._root, 'seqno')) + self.seqno = toolkit.Seqno(join(self._root, 'seqno')) for document in documents: if isinstance(document, basestring): @@ -74,34 +60,16 @@ class Volume(dict): def close(self): """Close operations with the server.""" _logger.info('Closing documents in %r', self._root) - + self._populators.kill() while self: __, cls = self.popitem() cls.close() - def connect(self, callback, condition=None): - self._subscriptions[callback] = condition or {} - - def disconnect(self, callback): - if callback in self._subscriptions: - del self._subscriptions[callback] - def populate(self): for cls in self.values(): for __ in cls.populate(): coroutine.dispatch() - def notify(self, event): - for callback, condition in self._subscriptions.items(): - for key, value in condition.items(): - if event.get(key) != value: - break - else: - try: - callback(event) - except Exception: - exception(_logger, 'Failed to dispatch %r', event) - def __enter__(self): return self @@ -111,273 +79,25 @@ class Volume(dict): def __getitem__(self, name): directory = self.get(name) enforce(directory is not None, http.BadRequest, - 'Unknown %r document', name) + 'Unknown %r resource', name) return directory - def _open(self, name, document): - if isinstance(document, basestring): - mod = __import__(document, fromlist=[name]) + def _open(self, name, resource): + if isinstance(resource, basestring): + mod = __import__(resource, fromlist=[name]) cls = getattr(mod, name.capitalize()) else: - cls = document + cls = resource directory = Directory(join(self._root, name), cls, self._index_class, - self.notify, self.seqno) + lambda event: self._broadcast(name, event), self.seqno) + self._populators.spawn(self._populate, directory) return directory + def _populate(self, directory): + for __ in directory.populate(): + coroutine.dispatch() -class VolumeCommands(CommandsProcessor): - - def __init__(self, volume): - CommandsProcessor.__init__(self, volume) - self.volume = volume - - @directory_command(method='POST', - permissions=env.ACCESS_AUTH, mime_type='application/json') - def create(self, request): - with self._post(request, env.ACCESS_CREATE) as (directory, doc): - event = {} - self.on_create(request, doc.props, event) - if 'guid' not in doc.props: - doc.props['guid'] = toolkit.uuid() - doc.guid = doc.props['guid'] - directory.create(doc.props, event) - return doc.guid - - @directory_command(method='GET', - arguments={'offset': to_int, 'limit': to_int, 'reply': to_list}, - mime_type='application/json') - def find(self, document, reply, request): - if not reply: - reply = ['guid'] - self._preget(request) - documents, total = self.volume[document].find(**request) - result = [self._get_props(i, request, reply) for i in documents] - return {'total': total, 'result': result} - - @document_command(method='GET', cmd='exists', - mime_type='application/json') - def exists(self, document, guid): - directory = self.volume[document] - return directory.exists(guid) - - @document_command(method='PUT', - permissions=env.ACCESS_AUTH | env.ACCESS_AUTHOR) - def update(self, request): - with self._post(request, env.ACCESS_WRITE) as (directory, doc): - if not doc.props: - return - event = {} - self.on_update(request, doc.props, event) - directory.update(doc.guid, doc.props, event) - - @property_command(method='PUT', - permissions=env.ACCESS_AUTH | env.ACCESS_AUTHOR) - def update_prop(self, request, prop, url=None): - if url: - value = PropertyMetadata(url=url) - elif request.content is None: - value = request.content_stream - else: - value = request.content - request.content = {prop: value} - self.update(request) - - @document_command(method='DELETE', - permissions=env.ACCESS_AUTH | env.ACCESS_AUTHOR) - def delete(self, request, document, guid): - directory = self.volume[document] - directory.delete(guid) - - @document_command(method='GET', arguments={'reply': to_list}, - mime_type='application/json') - def get(self, document, guid, reply, request): - if not reply: - reply = [] - for prop in self.volume[document].metadata.values(): - if prop.permissions & env.ACCESS_READ and \ - not (prop.permissions & env.ACCESS_LOCAL): - reply.append(prop.name) - self._preget(request) - doc = self.volume[document].get(guid) - return self._get_props(doc, request, reply) - - @property_command(method='GET', mime_type='application/json') - def get_prop(self, document, guid, prop, request, response): - directory = self.volume[document] - prop = directory.metadata[prop] - doc = directory.get(guid) - doc.request = request - - prop.assert_access(env.ACCESS_READ) - - if isinstance(prop, StoredProperty): - value = doc.get(prop.name, request.accept_language) - value = prop.on_get(doc, value) - if value is None: - value = prop.default - return value - else: - meta = prop.on_get(doc, doc.meta(prop.name)) - enforce(meta is not None and ('blob' in meta or 'url' in meta), - http.NotFound, 'BLOB does not exist') - return meta - - @property_command(method='HEAD') - def get_prop_meta(self, document, guid, prop, request, response): - directory = self.volume[document] - prop = directory.metadata[prop] - doc = directory.get(guid) - doc.request = request - - prop.assert_access(env.ACCESS_READ) - - if isinstance(prop, StoredProperty): - meta = doc.meta(prop.name) - value = meta.pop('value') - response.content_length = len(json.dumps(value)) - else: - meta = prop.on_get(doc, doc.meta(prop.name)) - enforce(meta is not None and ('blob' in meta or 'url' in meta), - http.NotFound, 'BLOB does not exist') - if 'blob' in meta: - meta.pop('blob') - meta['url'] = '/'.join([request.static_prefix] + request.path) - response.content_length = meta['blob_size'] - - response.meta.update(meta) - response.last_modified = meta['mtime'] - - def on_create(self, request, props, event): - if 'guid' in props: - # TODO Temporal security hole, see TODO - guid = props['guid'] - enforce(not self.volume[request['document']].exists(guid), - '%s already exists', guid) - enforce(_GUID_RE.match(guid) is not None, - 'Malformed %s GUID', guid) - ts = int(time.time()) - props['ctime'] = ts - props['mtime'] = ts - - def on_update(self, request, props, event): - props['mtime'] = int(time.time()) - - def after_post(self, doc): - pass - - @contextmanager - def _post(self, request, access): - enforce(isinstance(request.content, dict), 'Invalid value') - - directory = self.volume[request['document']] - if 'guid' in request: - doc = directory.get(request['guid']) - else: - doc = directory.document_class(None, {}) - doc.request = request - blobs = [] - - for name, value in request.content.items(): - prop = directory.metadata[name] - if isinstance(prop, BlobProperty): - prop.assert_access(env.ACCESS_CREATE if - access == env.ACCESS_WRITE and doc.meta(name) is None - else access) - if value is None: - value = {'blob': None} - elif isinstance(value, dict): - enforce('url' in value, - 'Key %r is not specified in %r blob property', - 'url', name) - value = {'url': value['url']} - else: - value = _read_blob(request, prop, value) - blobs.append(value['blob']) - else: - prop.assert_access(access) - if prop.localized and isinstance(value, basestring): - value = {request.accept_language[0]: value} - try: - value = prop.decode(value) - except Exception, error: - error = 'Value %r for %r property is invalid: %s' % \ - (value, prop.name, error) - exception(error) - raise RuntimeError(error) - doc[name] = value - - if access == env.ACCESS_CREATE: - for name, prop in directory.metadata.items(): - if not isinstance(prop, BlobProperty) and \ - name not in request.content and \ - (prop.default is not None or prop.on_set is not None): - doc[name] = prop.default - - try: - for name, value in doc.props.items(): - prop = directory.metadata[name] - if prop.on_set is not None: - doc.props[name] = prop.on_set(doc, value) - yield directory, doc - finally: - for path in blobs: - if exists(path): - os.unlink(path) - - self.after_post(doc) - - def _preget(self, request): - metadata = self.volume[request['document']].metadata - reply = request.setdefault('reply', []) - if reply: - for prop in reply: - metadata[prop].assert_access(env.ACCESS_READ) - else: - reply.append('guid') - - def _get_props(self, doc, request, props): - result = {} - metadata = doc.metadata - doc.request = request - for name in props: - prop = metadata[name] - value = prop.on_get(doc, doc.get(name, request.accept_language)) - if value is None: - value = prop.default - elif request.static_prefix and isinstance(value, PropertyMetadata): - value = value.get('url') - if value is None: - value = '/'.join(['', metadata.name, doc.guid, name]) - if value.startswith('/'): - value = request.static_prefix + value - result[name] = value - return result - - -def _read_blob(request, prop, value): - digest = hashlib.sha1() - dst = util.NamedTemporaryFile(delete=False) - - try: - if isinstance(value, basestring): - digest.update(value) - dst.write(value) - else: - size = request.content_length or sys.maxint - while size > 0: - chunk = value.read(min(size, BUFFER_SIZE)) - if not chunk: - break - dst.write(chunk) - size -= len(chunk) - digest.update(chunk) - except Exception: - os.unlink(dst.name) - raise - finally: - dst.close() - - return {'blob': dst.name, - 'digest': digest.hexdigest(), - 'mime_type': request.content_type or prop.mime_type, - } + def _broadcast(self, resource, event): + if self.broadcast is not None: + event['resource'] = resource + self.broadcast(event) diff --git a/sugar_network/resources/__init__.py b/sugar_network/model/__init__.py index dd22218..b0ba07a 100644 --- a/sugar_network/resources/__init__.py +++ b/sugar_network/model/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Aleksey Lim +# Copyright (C) 2012-2013 Aleksey Lim # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -13,6 +13,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +from sugar_network.model.routes import Routes + + CONTEXT_TYPES = ['activity', 'project', 'package', 'content'] NOTIFICATION_TYPES = ['create', 'update', 'delete', 'vote'] FEEDBACK_TYPES = ['question', 'idea', 'problem'] @@ -27,3 +30,16 @@ STABILITIES = [ ] RATINGS = [0, 1, 2, 3, 4, 5] + +RESOURCES = ( + 'sugar_network.model.artifact', + 'sugar_network.model.comment', + 'sugar_network.model.context', + 'sugar_network.model.implementation', + 'sugar_network.model.notification', + 'sugar_network.model.feedback', + 'sugar_network.model.report', + 'sugar_network.model.review', + 'sugar_network.model.solution', + 'sugar_network.model.user', + ) diff --git a/sugar_network/resources/artifact.py b/sugar_network/model/artifact.py index 90020c4..32ae506 100644 --- a/sugar_network/resources/artifact.py +++ b/sugar_network/model/artifact.py @@ -15,38 +15,38 @@ from os.path import join -from sugar_network import db, resources, static -from sugar_network.resources.volume import Resource +from sugar_network import db, model, static +from sugar_network.toolkit.router import Blob, ACL -class Artifact(Resource): +class Artifact(db.Resource): @db.indexed_property(prefix='C', - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + acl=ACL.CREATE | ACL.READ) def context(self, value): return value - @db.indexed_property(prefix='T', typecast=[resources.ARTIFACT_TYPES]) + @db.indexed_property(prefix='T', typecast=[model.ARTIFACT_TYPES]) def type(self, value): return value @db.indexed_property(slot=1, prefix='S', full_text=True, localized=True, - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + acl=ACL.CREATE | ACL.READ) def title(self, value): return value @db.indexed_property(prefix='D', full_text=True, localized=True, - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + acl=ACL.CREATE | ACL.READ) def description(self, value): return value - @db.indexed_property(slot=3, typecast=resources.RATINGS, default=0, - permissions=db.ACCESS_READ | db.ACCESS_CALC) + @db.indexed_property(slot=3, typecast=model.RATINGS, default=0, + acl=ACL.READ | ACL.CALC) def rating(self, value): return value @db.stored_property(typecast=[], default=[0, 0], - permissions=db.ACCESS_READ | db.ACCESS_CALC) + acl=ACL.READ | ACL.CALC) def reviews(self, value): if value is None: return 0 @@ -57,10 +57,11 @@ class Artifact(Resource): def preview(self, value): if value: return value - return db.PropertyMetadata( - url='/static/images/missing.png', - blob=join(static.PATH, 'images', 'missing.png'), - mime_type='image/png') + return Blob({ + 'url': '/static/images/missing.png', + 'blob': join(static.PATH, 'images', 'missing.png'), + 'mime_type': 'image/png', + }) @db.blob_property() def data(self, value): @@ -69,11 +70,11 @@ class Artifact(Resource): return value @db.indexed_property(prefix='K', typecast=bool, default=False, - permissions=db.ACCESS_READ | db.ACCESS_LOCAL) + acl=ACL.READ | ACL.LOCAL) def favorite(self, value): return value @db.indexed_property(prefix='L', typecast=[0, 1, 2], default=0, - permissions=db.ACCESS_READ | db.ACCESS_LOCAL) + acl=ACL.READ | ACL.LOCAL) def clone(self, value): return value diff --git a/sugar_network/resources/comment.py b/sugar_network/model/comment.py index a820979..88917ed 100644 --- a/sugar_network/resources/comment.py +++ b/sugar_network/model/comment.py @@ -14,18 +14,18 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. from sugar_network import db -from sugar_network.resources.volume import Resource +from sugar_network.toolkit.router import ACL -class Comment(Resource): +class Comment(db.Resource): @db.indexed_property(prefix='C', - permissions=db.ACCESS_READ) + acl=ACL.READ) def context(self, value): return value @db.indexed_property(prefix='R', default='', - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + acl=ACL.CREATE | ACL.READ) def review(self, value): return value @@ -37,7 +37,7 @@ class Comment(Resource): return value @db.indexed_property(prefix='F', default='', - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + acl=ACL.CREATE | ACL.READ) def feedback(self, value): return value @@ -49,7 +49,7 @@ class Comment(Resource): return value @db.indexed_property(prefix='S', default='', - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + acl=ACL.CREATE | ACL.READ) def solution(self, value): return value @@ -61,6 +61,6 @@ class Comment(Resource): return value @db.indexed_property(prefix='M', full_text=True, localized=True, - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + acl=ACL.CREATE | ACL.READ) def message(self, value): return value diff --git a/sugar_network/resources/context.py b/sugar_network/model/context.py index 4d30776..772a71a 100644 --- a/sugar_network/resources/context.py +++ b/sugar_network/model/context.py @@ -13,20 +13,16 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import logging from os.path import join -from sugar_network import db, resources, static -from sugar_network.resources.volume import Resource +from sugar_network import db, model, static +from sugar_network.toolkit.router import Blob, ACL -_logger = logging.getLogger('resources.context') - - -class Context(Resource): +class Context(db.Resource): @db.indexed_property(prefix='T', full_text=True, - typecast=[resources.CONTEXT_TYPES]) + typecast=[model.CONTEXT_TYPES]) def type(self, value): return value @@ -61,41 +57,44 @@ class Context(Resource): if value: return value if 'package' in self['type']: - return db.PropertyMetadata( - url='/static/images/package.png', - blob=join(static.PATH, 'images', 'package.png'), - mime_type='image/png') + return Blob({ + 'url': '/static/images/package.png', + 'blob': join(static.PATH, 'images', 'package.png'), + 'mime_type': 'image/png', + }) else: - return db.PropertyMetadata( - url='/static/images/missing.png', - blob=join(static.PATH, 'images', 'missing.png'), - mime_type='image/png') + return Blob({ + 'url': '/static/images/missing.png', + 'blob': join(static.PATH, 'images', 'missing.png'), + 'mime_type': 'image/png', + }) @db.blob_property(mime_type='image/svg+xml') def artifact_icon(self, value): if value: return value - return db.PropertyMetadata( - url='/static/images/missing.svg', - blob=join(static.PATH, 'images', 'missing.svg'), - mime_type='image/svg+xml') + return Blob({ + 'url': '/static/images/missing.svg', + 'blob': join(static.PATH, 'images', 'missing.svg'), + 'mime_type': 'image/svg+xml', + }) @db.blob_property(mime_type='image/png') def preview(self, value): if value: return value - return db.PropertyMetadata( - url='/static/images/missing.png', - blob=join(static.PATH, 'images', 'missing.png'), - mime_type='image/png') - - @db.indexed_property(slot=3, typecast=resources.RATINGS, default=0, - permissions=db.ACCESS_READ | db.ACCESS_CALC) + return Blob({ + 'url': '/static/images/missing.png', + 'blob': join(static.PATH, 'images', 'missing.png'), + 'mime_type': 'image/png', + }) + + @db.indexed_property(slot=3, typecast=model.RATINGS, default=0, + acl=ACL.READ | ACL.CALC) def rating(self, value): return value - @db.stored_property(typecast=[], default=[0, 0], - permissions=db.ACCESS_READ | db.ACCESS_CALC) + @db.stored_property(typecast=[], default=[0, 0], acl=ACL.READ | ACL.CALC) def reviews(self, value): if value is None: return 0 @@ -103,17 +102,16 @@ class Context(Resource): return value[0] @db.indexed_property(prefix='K', typecast=bool, default=False, - permissions=db.ACCESS_READ | db.ACCESS_LOCAL) + acl=ACL.READ | ACL.LOCAL) def favorite(self, value): return value @db.indexed_property(prefix='L', typecast=[0, 1, 2], default=0, - permissions=db.ACCESS_READ | db.ACCESS_LOCAL) + acl=ACL.READ | ACL.LOCAL) def clone(self, value): return value - @db.stored_property(typecast=[], default=[], - permissions=db.ACCESS_PUBLIC | db.ACCESS_LOCAL) + @db.stored_property(typecast=[], default=[], acl=ACL.PUBLIC | ACL.LOCAL) def dependencies(self, value): """Software dependencies. @@ -124,11 +122,10 @@ class Context(Resource): return value @db.stored_property(typecast=dict, default={}, - permissions=db.ACCESS_PUBLIC | db.ACCESS_LOCAL) + acl=ACL.PUBLIC | ACL.LOCAL) def aliases(self, value): return value - @db.stored_property(typecast=dict, default={}, - permissions=db.ACCESS_PUBLIC | db.ACCESS_LOCAL | db.ACCESS_SYSTEM) + @db.stored_property(typecast=dict, default={}, acl=ACL.PUBLIC | ACL.LOCAL) def packages(self, value): return value diff --git a/sugar_network/resources/feedback.py b/sugar_network/model/feedback.py index 97ba348..45bacfe 100644 --- a/sugar_network/resources/feedback.py +++ b/sugar_network/model/feedback.py @@ -13,18 +13,17 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from sugar_network import db, resources -from sugar_network.resources.volume import Resource +from sugar_network import db, model +from sugar_network.toolkit.router import ACL -class Feedback(Resource): +class Feedback(db.Resource): - @db.indexed_property(prefix='C', - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + @db.indexed_property(prefix='C', acl=ACL.CREATE | ACL.READ) def context(self, value): return value - @db.indexed_property(prefix='T', typecast=[resources.FEEDBACK_TYPES]) + @db.indexed_property(prefix='T', typecast=[model.FEEDBACK_TYPES]) def type(self, value): return value diff --git a/sugar_network/resources/implementation.py b/sugar_network/model/implementation.py index d915473..d0f03f8 100644 --- a/sugar_network/resources/implementation.py +++ b/sugar_network/model/implementation.py @@ -15,17 +15,17 @@ import xapian -from sugar_network import db, resources -from sugar_network.resources.volume import Resource +from sugar_network import db, model +from sugar_network.toolkit.router import ACL from sugar_network.toolkit.licenses import GOOD_LICENSES from sugar_network.toolkit.spec import parse_version from sugar_network.toolkit import http, enforce -class Implementation(Resource): +class Implementation(db.Resource): @db.indexed_property(prefix='C', - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + acl=ACL.CREATE | ACL.READ) def context(self, value): return value @@ -37,24 +37,23 @@ class Implementation(Resource): return value @db.indexed_property(prefix='L', full_text=True, typecast=[GOOD_LICENSES], - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + acl=ACL.CREATE | ACL.READ) def license(self, value): return value - @db.indexed_property(slot=1, prefix='V', - reprcast=lambda x: _encode_version(x), - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + @db.indexed_property(slot=1, prefix='V', fmt=lambda x: _fmt_version(x), + acl=ACL.CREATE | ACL.READ) def version(self, value): return value @db.indexed_property(prefix='S', - permissions=db.ACCESS_CREATE | db.ACCESS_READ, - typecast=resources.STABILITIES) + acl=ACL.CREATE | ACL.READ, + typecast=model.STABILITIES) def stability(self, value): return value @db.indexed_property(prefix='N', full_text=True, localized=True, - default='', permissions=db.ACCESS_CREATE | db.ACCESS_READ) + default='', acl=ACL.CREATE | ACL.READ) def notes(self, value): return value @@ -63,7 +62,7 @@ class Implementation(Resource): return value -def _encode_version(version): +def _fmt_version(version): version = parse_version(version) # Convert to [(`version`, `modifier`)] version = zip(*([iter(version)] * 2)) diff --git a/sugar_network/resources/notification.py b/sugar_network/model/notification.py index 73878df..e567b65 100644 --- a/sugar_network/resources/notification.py +++ b/sugar_network/model/notification.py @@ -13,35 +13,35 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from sugar_network import db, resources -from sugar_network.resources.volume import Resource +from sugar_network import db, model +from sugar_network.toolkit.router import ACL -class Notification(Resource): +class Notification(db.Resource): @db.indexed_property(prefix='T', - permissions=db.ACCESS_CREATE | db.ACCESS_READ, - typecast=resources.NOTIFICATION_TYPES) + acl=ACL.CREATE | ACL.READ, + typecast=model.NOTIFICATION_TYPES) def type(self, value): return value @db.indexed_property(prefix='K', - permissions=db.ACCESS_CREATE | db.ACCESS_READ, - default='', typecast=resources.NOTIFICATION_OBJECT_TYPES) + acl=ACL.CREATE | ACL.READ, + default='', typecast=model.NOTIFICATION_OBJECT_TYPES) def resource(self, value): return value @db.indexed_property(prefix='O', - permissions=db.ACCESS_CREATE | db.ACCESS_READ, default='') + acl=ACL.CREATE | ACL.READ, default='') def object(self, value): return value @db.indexed_property(prefix='D', - permissions=db.ACCESS_CREATE | db.ACCESS_READ, default='') + acl=ACL.CREATE | ACL.READ, default='') def to(self, value): return value @db.indexed_property(prefix='M', full_text=True, localized=True, - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + acl=ACL.CREATE | ACL.READ) def message(self, value): return value diff --git a/sugar_network/resources/report.py b/sugar_network/model/report.py index ed5cdf7..8b2ca05 100644 --- a/sugar_network/resources/report.py +++ b/sugar_network/model/report.py @@ -14,18 +14,16 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. from sugar_network import db -from sugar_network.resources.volume import Resource +from sugar_network.toolkit.router import ACL -class Report(Resource): +class Report(db.Resource): - @db.indexed_property(prefix='C', - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + @db.indexed_property(prefix='C', acl=ACL.CREATE | ACL.READ) def context(self, value): return value - @db.indexed_property(prefix='V', - permissions=db.ACCESS_CREATE | db.ACCESS_READ, default='') + @db.indexed_property(prefix='V', default='', acl=ACL.CREATE | ACL.READ) def implementation(self, value): return value @@ -36,31 +34,18 @@ class Report(Resource): self['version'] = version['version'] return value - @db.stored_property(default='', - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + @db.stored_property(default='', acl=ACL.CREATE | ACL.READ) def version(self, value): return value - @db.stored_property(typecast=dict, default={}, - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + @db.stored_property(typecast=dict, default={}, acl=ACL.CREATE | ACL.READ) def environ(self, value): return value - @db.indexed_property(prefix='T', - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + @db.indexed_property(prefix='T', acl=ACL.CREATE | ACL.READ) def error(self, value): return value @db.blob_property() def data(self, value): return value - - @db.document_command(method='GET', cmd='log', - mime_type='text/html') - def log(self, guid): - # In further implementations, `data` might be a tarball - data = self.meta('data') - if data and 'blob' in data: - return file(data['blob'], 'rb') - else: - return '' diff --git a/sugar_network/resources/review.py b/sugar_network/model/review.py index e8274cc..621ac87 100644 --- a/sugar_network/resources/review.py +++ b/sugar_network/model/review.py @@ -13,19 +13,19 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from sugar_network import db, resources -from sugar_network.resources.volume import Resource +from sugar_network import db, model +from sugar_network.toolkit.router import ACL -class Review(Resource): +class Review(db.Resource): @db.indexed_property(prefix='C', - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + acl=ACL.CREATE | ACL.READ) def context(self, value): return value @db.indexed_property(prefix='A', default='', - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + acl=ACL.CREATE | ACL.READ) def artifact(self, value): return value @@ -37,16 +37,16 @@ class Review(Resource): return value @db.indexed_property(prefix='S', full_text=True, localized=True, - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + acl=ACL.CREATE | ACL.READ) def title(self, value): return value @db.indexed_property(prefix='N', full_text=True, localized=True, - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + acl=ACL.CREATE | ACL.READ) def content(self, value): return value - @db.indexed_property(slot=1, typecast=resources.RATINGS, - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + @db.indexed_property(slot=1, typecast=model.RATINGS, + acl=ACL.CREATE | ACL.READ) def rating(self, value): return value diff --git a/sugar_network/model/routes.py b/sugar_network/model/routes.py new file mode 100644 index 0000000..dc92554 --- /dev/null +++ b/sugar_network/model/routes.py @@ -0,0 +1,165 @@ +# Copyright (C) 2013 Aleksey Lim +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import json +import logging +import mimetypes +from os.path import join, split + +from sugar_network import static +from sugar_network.toolkit.router import route, fallbackroute, Blob, ACL +from sugar_network.toolkit import coroutine + + +_logger = logging.getLogger('model.routes') + + +class Routes(object): + + def __init__(self): + self._pooler = _Pooler() + + @route('GET', mime_type='text/html') + def hello(self): + return _HELLO_HTML + + @route('OPTIONS') + def options(self, request, response): + if request.environ['HTTP_ORIGIN']: + response['Access-Control-Allow-Methods'] = \ + request.environ['HTTP_ACCESS_CONTROL_REQUEST_METHOD'] + response['Access-Control-Allow-Headers'] = \ + request.environ['HTTP_ACCESS_CONTROL_REQUEST_HEADERS'] + else: + response['Allow'] = 'GET, HEAD, POST, PUT, DELETE' + response.content_length = 0 + + @route('GET', cmd='whoami', mime_type='application/json') + def whoami(self, request, response): + roles = [] + # pylint: disable-msg=E1101 + if self.authorize(request.principal, 'root'): + roles.append('root') + return {'roles': roles, 'guid': request.principal} + + @route('GET', cmd='subscribe', mime_type='text/event-stream') + def subscribe(self, request=None, response=None, ping=False, **condition): + """Subscribe to Server-Sent Events.""" + if request is not None and not condition: + condition = request + if response is not None: + response.content_type = 'text/event-stream' + response['Cache-Control'] = 'no-cache' + peer = 'anonymous' + if hasattr(request, 'environ'): + peer = request.environ.get('HTTP_X_SN_LOGIN') or peer + return self._pull_events(peer, ping, condition) + + @route('POST', cmd='broadcast', + mime_type='application/json', acl=ACL.LOCAL) + def broadcast(self, event=None, request=None): + if request is not None: + event = request.content + _logger.debug('Broadcast event: %r', event) + self._pooler.notify_all(event) + coroutine.dispatch() + + @fallbackroute('GET', ['static']) + def get_static(self, request): + path = join(static.PATH, *request.path[1:]) + if not mimetypes.inited: + mimetypes.init() + mime_type = mimetypes.types_map.get('.' + path.rsplit('.', 1)[-1]) + return Blob({ + 'blob': path, + 'filename': split(path)[-1], + 'mime_type': mime_type, + }) + + @route('GET', ['robots.txt'], mime_type='text/plain') + def robots(self, request, response): + return 'User-agent: *\nDisallow: /\n' + + @route('GET', ['favicon.ico']) + def favicon(self, request, response): + return Blob({ + 'blob': join(static.PATH, 'favicon.ico'), + 'mime_type': 'image/x-icon', + }) + + def _pull_events(self, peer, ping, condition): + _logger.debug('Start pulling events to %s user', peer) + + if ping: + # XXX The whole commands' kwargs handling should be redesigned + if 'ping' in condition: + condition.pop('ping') + # If non-greenlet application needs only to initiate + # a subscription and do not stuck in waiting for the first event, + # it should pass `ping` argument to return fake event to unblock + # `GET /?cmd=subscribe` call. + yield 'data: %s\n\n' % json.dumps({'event': 'pong'}) + + try: + while True: + event = self._pooler.wait() + for key, value in condition.items(): + if value.startswith('!'): + if event.get(key) == value[1:]: + break + elif event.get(key) != value: + break + else: + yield 'data: %s\n\n' % json.dumps(event) + finally: + _logger.debug('Stop pulling events to %s user', peer) + + +class _Pooler(object): + """One-producer-to-many-consumers events delivery.""" + + def __init__(self): + self._value = None + self._waiters = 0 + self._ready = coroutine.Event() + self._open = coroutine.Event() + self._open.set() + + def wait(self): + self._open.wait() + self._waiters += 1 + try: + self._ready.wait() + finally: + self._waiters -= 1 + if self._waiters == 0: + self._ready.clear() + self._open.set() + return self._value + + def notify_all(self, value=None): + self._open.wait() + if not self._waiters: + return + self._open.clear() + self._value = value + self._ready.set() + + +_HELLO_HTML = """\ +<h2>Welcome to Sugar Network API!</h2> +Visit the <a href="http://wiki.sugarlabs.org/go/Sugar_Network/API"> +Sugar Labs Wiki</a> to learn how it can be used. +""" diff --git a/sugar_network/resources/solution.py b/sugar_network/model/solution.py index 6592b25..549e0f2 100644 --- a/sugar_network/resources/solution.py +++ b/sugar_network/model/solution.py @@ -14,18 +14,16 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. from sugar_network import db -from sugar_network.resources.volume import Resource +from sugar_network.toolkit.router import ACL -class Solution(Resource): +class Solution(db.Resource): - @db.indexed_property(prefix='C', - permissions=db.ACCESS_READ) + @db.indexed_property(prefix='C', acl=ACL.READ) def context(self, value): return value - @db.indexed_property(prefix='P', - permissions=db.ACCESS_CREATE | db.ACCESS_READ) + @db.indexed_property(prefix='P', acl=ACL.CREATE | ACL.READ) def feedback(self, value): return value diff --git a/sugar_network/resources/solution.py b/sugar_network/model/user.py index 6592b25..13c8684 100644 --- a/sugar_network/resources/solution.py +++ b/sugar_network/model/user.py @@ -14,28 +14,35 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. from sugar_network import db -from sugar_network.resources.volume import Resource +from sugar_network.toolkit.router import ACL -class Solution(Resource): +class User(db.Resource): - @db.indexed_property(prefix='C', - permissions=db.ACCESS_READ) - def context(self, value): + @db.indexed_property(slot=1, prefix='N', full_text=True) + def name(self, value): return value - @db.indexed_property(prefix='P', - permissions=db.ACCESS_CREATE | db.ACCESS_READ) - def feedback(self, value): + @db.stored_property() + def color(self, value): return value - @feedback.setter - def feedback(self, value): - if value: - feedback = self.volume['feedback'].get(value) - self['context'] = feedback['context'] + @db.indexed_property(prefix='S', default='', acl=ACL.CREATE | ACL.WRITE) + def machine_sn(self, value): return value - @db.indexed_property(prefix='N', full_text=True, localized=True) - def content(self, value): + @db.indexed_property(prefix='U', default='', acl=ACL.CREATE | ACL.WRITE) + def machine_uuid(self, value): + return value + + @db.stored_property(acl=ACL.CREATE) + def pubkey(self, value): + return value + + @db.indexed_property(prefix='P', full_text=True, default='') + def location(self, value): + return value + + @db.indexed_property(slot=2, prefix='B', default=0, typecast=int) + def birthday(self, value): return value diff --git a/sugar_network/node/__init__.py b/sugar_network/node/__init__.py index e5fa2e1..6b1588f 100644 --- a/sugar_network/node/__init__.py +++ b/sugar_network/node/__init__.py @@ -47,10 +47,6 @@ find_limit = Option( 'limit the resulting list for search requests', default=32, type_cast=int, name='find-limit') -static_url = Option( - 'url prefix to use for static files that should be served via API ' - 'server; if omited, HTTP_HOST request value will be used') - stats_root = Option( 'path to the root directory for placing stats', default='/var/lib/sugar-network/stats', name='stats_root') diff --git a/sugar_network/node/auth.py b/sugar_network/node/auth.py deleted file mode 100644 index fa77efa..0000000 --- a/sugar_network/node/auth.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (C) 2012 Aleksey Lim -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -import os -from ConfigParser import ConfigParser -from os.path import join, exists - -from sugar_network import node -from sugar_network.toolkit import http, enforce - - -_config_mtime = 0 -_config = None - - -def validate(request, role): - enforce(_validate(request, role), http.Forbidden, - 'No enough permissions to proceed the operation') - - -def try_validate(request, role): - return _validate(request, role) or False - - -def reset(): - global _config_mtime - _config_mtime = 0 - - -def _validate(request, role): - global _config_mtime, _config - - if role == 'user': - if request.principal: - return True - - config_path = join(node.data_root.value, 'authorization.conf') - if exists(config_path): - mtime = os.stat(config_path).st_mtime - if mtime > _config_mtime: - _config_mtime = mtime - _config = ConfigParser() - _config.read(config_path) - if _config is None: - return - - user = request.principal or 'anonymous' - if not _config.has_section(user): - user = 'DEFAULT' - - if _config.has_option(user, role): - return _config.get(user, role).strip().lower() in \ - ('true', 'on', '1', 'allow') diff --git a/sugar_network/node/files.py b/sugar_network/node/files.py index 4694d94..4fb64ca 100644 --- a/sugar_network/node/files.py +++ b/sugar_network/node/files.py @@ -20,7 +20,8 @@ from bisect import bisect_left from shutil import copyfileobj from os.path import join, exists, relpath, lexists, dirname -from sugar_network.toolkit import util, coroutine +from sugar_network import toolkit +from sugar_network.toolkit import coroutine _logger = logging.getLogger('node.sync_files') @@ -38,7 +39,7 @@ def merge(files_path, packet): path = join(files_path, record['path']) if not exists(dirname(path)): os.makedirs(dirname(path)) - with util.new_file(path) as f: + with toolkit.new_file(path) as f: copyfileobj(record['blob'], f) elif op == 'delete': path = join(files_path, record['path']) @@ -73,7 +74,7 @@ class Index(object): def diff(self, in_seq, out_seq=None, **kwargs): if out_seq is None: - out_seq = util.Sequence([]) + out_seq = toolkit.Sequence([]) is_initial_diff = not out_seq # Below calls will trigger coroutine switches, thius, @@ -103,7 +104,7 @@ class Index(object): yield {'op': 'update', 'path': path, 'blob_size': os.stat(blob_path).st_size, - 'blob': util.iter_file(blob_path), + 'blob': toolkit.iter_file(blob_path), } out_seq.include(start, seqno) start = seqno @@ -176,7 +177,7 @@ class Index(object): self._stamp = os.stat(self._files_path).st_mtime if self._seqno.commit(): - with util.new_file(self._index_path) as f: + with toolkit.new_file(self._index_path) as f: json.dump((self._index, self._stamp), f) return True diff --git a/sugar_network/node/master.py b/sugar_network/node/master.py index 3619520..b2a7630 100644 --- a/sugar_network/node/master.py +++ b/sugar_network/node/master.py @@ -20,10 +20,11 @@ import logging from Cookie import SimpleCookie from os.path import join -from sugar_network import db, node +from sugar_network import node, toolkit from sugar_network.node import sync, stats_user, files, volume, downloads, obs -from sugar_network.node.commands import NodeCommands -from sugar_network.toolkit import http, cachedir, coroutine, util, enforce +from sugar_network.node.routes import NodeRoutes +from sugar_network.toolkit.router import route, ACL +from sugar_network.toolkit import http, coroutine, enforce _ONE_WAY_DOCUMENTS = ['report'] @@ -31,10 +32,10 @@ _ONE_WAY_DOCUMENTS = ['report'] _logger = logging.getLogger('node.master') -class MasterCommands(NodeCommands): +class MasterRoutes(NodeRoutes): def __init__(self, guid, volume_): - NodeCommands.__init__(self, guid, volume_) + NodeRoutes.__init__(self, guid, volume_) self._pulls = { 'pull': lambda **kwargs: @@ -44,15 +45,16 @@ class MasterCommands(NodeCommands): ('files_diff', None, self._files.diff(**kwargs)), } - self._pull_queue = downloads.Pool(join(cachedir.value, 'pulls')) + self._pull_queue = downloads.Pool( + join(toolkit.cachedir.value, 'pulls')) self._files = None if node.files_root.value: self._files = files.Index(node.files_root.value, join(volume_.root, 'files.index'), volume_.seqno) - @db.volume_command(method='POST', cmd='sync', - permissions=db.ACCESS_AUTH) + @route('POST', cmd='sync', + acl=ACL.AUTH) def sync(self, request): reply, cookie = self._push(sync.decode(request.content_stream)) exclude_seq = None @@ -63,7 +65,7 @@ class MasterCommands(NodeCommands): exclude_seq=exclude_seq, layer=layer)) return sync.encode(reply, src=self.guid) - @db.volume_command(method='POST', cmd='push') + @route('POST', cmd='push') def push(self, request, response): reply, cookie = self._push(sync.package_decode(request.content_stream)) # Read passed cookie only after excluding `merged_seq`. @@ -73,19 +75,19 @@ class MasterCommands(NodeCommands): cookie.store(response) return sync.package_encode(reply, src=self.guid) - @db.volume_command(method='GET', cmd='pull', + @route('GET', cmd='pull', mime_type='application/octet-stream', - arguments={'accept_length': db.to_int}) + arguments={'accept_length': int}) def pull(self, request, response, accept_length=None): cookie = _Cookie(request) if not cookie: _logger.warning('Requested full dump in pull command') - cookie.append(('pull', None, util.Sequence([[1, None]]))) - cookie.append(('files_pull', None, util.Sequence([[1, None]]))) + cookie.append(('pull', None, toolkit.Sequence([[1, None]]))) + cookie.append(('files_pull', None, toolkit.Sequence([[1, None]]))) exclude_seq = None if len(cookie.sent) == 1: - exclude_seq = util.Sequence(cookie.sent.values()[0]) + exclude_seq = toolkit.Sequence(cookie.sent.values()[0]) reply = None for pull_key in cookie: @@ -110,7 +112,7 @@ class MasterCommands(NodeCommands): _logger.debug('Existing %r is too big, will recreate', pull) self._pull_queue.remove(pull_key) - out_seq = util.Sequence() + out_seq = toolkit.Sequence() pull = self._pull_queue.set(pull_key, out_seq, sync.sneakernet_encode, [self._pulls[op](in_seq=seq, out_seq=out_seq, @@ -130,15 +132,13 @@ class MasterCommands(NodeCommands): cookie.store(response) return reply - @db.document_command(method='PUT', cmd='presolve', - permissions=db.ACCESS_AUTH, mime_type='application/json') - def presolve(self, request, document, guid): - enforce(document == 'context', http.BadRequest, - 'Only Contexts can be presolved') + @route('PUT', ['context', None], cmd='presolve', + acl=ACL.AUTH, mime_type='application/json') + def presolve(self, request): enforce(node.files_root.value, http.BadRequest, 'Disabled') - package = self.volume[document].get(guid) - enforce(package['aliases'], http.BadRequest, 'Nothing to presolve') - return obs.presolve(package['aliases'], node.files_root.value) + aliases = self.volume['context'].get(request.guid)['aliases'] + enforce(aliases, http.BadRequest, 'Nothing to presolve') + return obs.presolve(aliases, node.files_root.value) def after_post(self, doc): if doc.metadata.name == 'context': @@ -150,7 +150,7 @@ class MasterCommands(NodeCommands): if shift_implementations and not doc.is_new: # Shift mtime to invalidate solutions self.volume['implementation'].mtime = int(time.time()) - NodeCommands.after_post(self, doc) + NodeRoutes.after_post(self, doc) def _push(self, stream): reply = [] @@ -163,7 +163,7 @@ class MasterCommands(NodeCommands): if packet.name == 'pull': pull_seq = cookie['pull', packet['layer'] or None] pull_seq.include(packet['sequence']) - cookie.sent.setdefault(src, util.Sequence()) + cookie.sent.setdefault(src, toolkit.Sequence()) elif packet.name == 'files_pull': if self._files is not None: cookie['files_pull'].include(packet['sequence']) @@ -174,7 +174,7 @@ class MasterCommands(NodeCommands): 'sequence': seq, 'dst': src, }, None)) - sent_seq = cookie.sent.setdefault(src, util.Sequence()) + sent_seq = cookie.sent.setdefault(src, toolkit.Sequence()) sent_seq.include(ack_seq) elif packet.name == 'stats_diff': reply.append(('stats_ack', { @@ -216,8 +216,7 @@ class MasterCommands(NodeCommands): package['status'] = 'no packages to resolve' if packages != doc['packages']: - doc.request.call('PUT', document='context', guid=doc.guid, - content={'packages': packages}) + self.volume['context'].update(doc.guid, {'packages': packages}) if node.files_root.value: obs.presolve(doc['aliases'], node.files_root.value) @@ -262,7 +261,7 @@ class _Cookie(list): for op, layer, seq in self: if (op, layer) == key: return seq - seq = util.Sequence() + seq = toolkit.Sequence() self.append(key + (seq,)) return seq diff --git a/sugar_network/node/obs.py b/sugar_network/node/obs.py index 6b99620..753e59e 100644 --- a/sugar_network/node/obs.py +++ b/sugar_network/node/obs.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Aleksey Lim +# Copyright (C) 2012-2013 Aleksey Lim # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,7 +19,8 @@ import logging from xml.etree import cElementTree as ElementTree from os.path import join, exists, basename -from sugar_network.toolkit import Option, http, util, exception, enforce +from sugar_network import toolkit +from sugar_network.toolkit import Option, http, enforce obs_url = Option( @@ -81,7 +82,7 @@ def presolve(aliases, dst_path): binaries.append(dict(pkg.items())) presolves.append((package, binaries)) except Exception: - exception(_logger, 'Failed to presolve %r on %s', + toolkit.exception(_logger, 'Failed to presolve %r on %s', names, repo['name']) continue @@ -107,7 +108,7 @@ def presolve(aliases, dst_path): files.append(binary) for package, info in result.items(): - with util.new_file(join(dst_dir, package)) as f: + with toolkit.new_file(join(dst_dir, package)) as f: json.dump(info, f) return {'repo': repo['name'], 'packages': result} diff --git a/sugar_network/node/commands.py b/sugar_network/node/routes.py index d0555b9..9a416d8 100644 --- a/sugar_network/node/commands.py +++ b/sugar_network/node/routes.py @@ -16,90 +16,49 @@ import os import logging import hashlib +from ConfigParser import ConfigParser from os.path import join, isdir, exists -from sugar_network import db, node, static -from sugar_network.node import auth, stats_node -from sugar_network.resources.volume import Commands +from sugar_network import db, node, toolkit, model +from sugar_network.node import stats_node, stats_user +from sugar_network.toolkit.router import route, preroute, postroute +from sugar_network.toolkit.router import ACL, fallbackroute from sugar_network.toolkit.spec import parse_requires, ensure_requires -from sugar_network.toolkit import http, util, coroutine, exception, enforce +from sugar_network.toolkit import http, coroutine, enforce _MAX_STATS_LENGTH = 100 -_logger = logging.getLogger('node.commands') +_logger = logging.getLogger('node.routes') -class NodeCommands(db.VolumeCommands, Commands): +class NodeRoutes(db.Routes, model.Routes): def __init__(self, guid, volume): - db.VolumeCommands.__init__(self, volume) - Commands.__init__(self) + db.Routes.__init__(self, volume) + model.Routes.__init__(self) + volume.broadcast = self.broadcast self._guid = guid self._stats = None + self._authenticated = set() + self._auth_config = None + self._auth_config_mtime = 0 if stats_node.stats_node.value: self._stats = stats_node.Sniffer(volume) coroutine.spawn(self._commit_stats) - self.volume.connect(self.broadcast) - @property def guid(self): return self._guid - @db.route('GET', '/robots.txt') - def robots(self, request, response): - response.content_type = 'text/plain' - return 'User-agent: *\nDisallow: /\n' - - @db.route('GET', '/favicon.ico') - def favicon(self, request, response): - return db.PropertyMetadata( - blob=join(static.PATH, 'favicon.ico'), - mime_type='image/x-icon') - - @db.route('GET', '/packages') - def route_packages(self, request, response): - enforce(node.files_root.value, http.BadRequest, 'Disabled') - if request.path and request.path[-1] == 'updates': - root = join(node.files_root.value, *request.path[:-1]) - enforce(isdir(root), http.NotFound, 'Directory was not found') - result = [] - last_modified = 0 - for filename in os.listdir(root): - if '.' in filename: - continue - path = join(root, filename) - mtime = os.stat(path).st_mtime - if mtime > request.if_modified_since: - result.append(filename) - last_modified = max(last_modified, mtime) - response.content_type = 'application/json' - if last_modified: - response.last_modified = last_modified - return result - else: - path = join(node.files_root.value, *request.path) - enforce(exists(path), http.NotFound, 'File was not found') - if isdir(path): - response.content_type = 'application/json' - return os.listdir(path) - else: - return util.iter_file(path) - - @db.volume_command(method='GET') - def hello(self, request, response): - raise http.Redirect('http://wiki.sugarlabs.org/go/Sugar_Network/API') - - @db.volume_command(method='GET', cmd='stat', + @route('GET', cmd='status', mime_type='application/json') - def stat(self): - # TODO Remove, it is deprecated - return self.info() + def status(self): + return {'route': 'direct'} - @db.volume_command(method='GET', cmd='info', + @route('GET', cmd='info', mime_type='application/json') def info(self): documents = {} @@ -107,13 +66,9 @@ class NodeCommands(db.VolumeCommands, Commands): documents[name] = {'mtime': directory.mtime} return {'guid': self._guid, 'documents': documents} - @db.volume_command(method='GET', cmd='stats', - mime_type='application/json', arguments={ - 'start': db.to_int, - 'end': db.to_int, - 'resolution': db.to_int, - 'source': db.to_list, - }) + @route('GET', cmd='stats', arguments={ + 'start': int, 'end': int, 'resolution': int, 'source': list}, + mime_type='application/json') def stats(self, start, end, resolution, source): if not source: return {} @@ -147,66 +102,77 @@ class NodeCommands(db.VolumeCommands, Commands): return result - @db.document_command(method='DELETE', - permissions=db.ACCESS_AUTH | db.ACCESS_AUTHOR) - def delete(self, request, document, guid): + @fallbackroute('GET', ['packages']) + def route_packages(self, request, response): + enforce(node.files_root.value, http.BadRequest, 'Disabled') + if request.path and request.path[-1] == 'updates': + root = join(node.files_root.value, *request.path[:-1]) + enforce(isdir(root), http.NotFound, 'Directory was not found') + result = [] + last_modified = 0 + for filename in os.listdir(root): + if '.' in filename: + continue + path = join(root, filename) + mtime = os.stat(path).st_mtime + if mtime > request.if_modified_since: + result.append(filename) + last_modified = max(last_modified, mtime) + response.content_type = 'application/json' + if last_modified: + response.last_modified = last_modified + return result + else: + path = join(node.files_root.value, *request.path) + enforce(exists(path), http.NotFound, 'File was not found') + if isdir(path): + response.content_type = 'application/json' + return os.listdir(path) + else: + return toolkit.iter_file(path) + + @route('DELETE', [None, None], acl=ACL.AUTH | ACL.AUTHOR) + def delete(self, request): # Servers data should not be deleted immediately # to let master-slave synchronization possible - request['method'] = 'PUT' + request.method = 'PUT' request.content = {'layer': ['deleted']} self.update(request) - @db.document_command(method='PUT', cmd='attach', - permissions=db.ACCESS_AUTH) - def attach(self, document, guid, request): - auth.validate(request, 'root') - directory = self.volume[document] - doc = directory.get(guid) + @route('PUT', [None, None], cmd='attach', acl=ACL.AUTH | ACL.SUPERUSER) + def attach(self, request): # TODO Reading layer here is a race + directory = self.volume[request.resource] + doc = directory.get(request.guid) layer = list(set(doc['layer']) | set(request.content)) - directory.update(guid, {'layer': layer}) - - @db.document_command(method='PUT', cmd='detach', - permissions=db.ACCESS_AUTH) - def detach(self, document, guid, request): - auth.validate(request, 'root') - directory = self.volume[document] - doc = directory.get(guid) + directory.update(request.guid, {'layer': layer}) + + @route('PUT', [None, None], cmd='detach', acl=ACL.AUTH | ACL.SUPERUSER) + def detach(self, request): # TODO Reading layer here is a race + directory = self.volume[request.resource] + doc = directory.get(request.guid) layer = list(set(doc['layer']) - set(request.content)) - directory.update(guid, {'layer': layer}) + directory.update(request.guid, {'layer': layer}) - @db.volume_command(method='GET', cmd='status', - mime_type='application/json') - def status(self): - return {'route': 'direct'} - - @db.volume_command(method='GET', cmd='whoami', - mime_type='application/json') - def whoami(self, request): - roles = [] - if self.validate(request, 'root'): - roles.append('root') - return {'roles': roles, 'guid': request.principal} - - @db.document_command(method='GET', cmd='clone', - arguments={'requires': db.to_list}) + @route('GET', ['context', None], cmd='clone', + arguments={'requires': list}) def clone(self, request, response): impl = self._clone(request) - return self.get_prop('implementation', impl.guid, 'data', - request, response) + request.path = ['implementation', impl.guid, 'data'] + return self.get_prop(request) - @db.document_command(method='HEAD', cmd='clone', - arguments={'requires': db.to_list}) + @route('HEAD', ['context', None], cmd='clone', + arguments={'requires': list}) def meta_clone(self, request, response): impl = self._clone(request) props = impl.properties(['guid', 'license', 'version', 'stability']) response.meta.update(props) response.meta.update(impl.meta('data')['spec']['*-*']) - @db.document_command(method='GET', cmd='deplist', - mime_type='application/json', arguments={'requires': db.to_list}) - def deplist(self, document, guid, repo, layer, requires, + @route('GET', ['context', None], cmd='deplist', + mime_type='application/json', arguments={'requires': list}) + def deplist(self, request, repo, layer, requires, stability='stable'): """List of native packages context is dependening on. @@ -220,17 +186,18 @@ class NodeCommands(db.VolumeCommands, Commands): list of package names """ - enforce(document == 'context') enforce(repo, 'Argument %r should be set', 'repo') - impls, total = self.volume['implementation'].find(context=guid, + impls, total = self.volume['implementation'].find(context=request.guid, layer=layer, stability=stability, requires=requires, order_by='-version', limit=1) enforce(total, http.NotFound, 'No implementations') result = [] - for package in set(next(impls)['spec']['*-*'].get('requires') or []) \ - | set(self.volume['context'].get(guid)['dependencies']): + common_deps = self.volume['context'].get(request.guid)['dependencies'] + spec = next(impls).meta('data')['spec']['*-*'] + + for package in set(spec.get('requires') or []) | set(common_deps): if package == 'sugar': continue dep = self.volume['context'].get(package) @@ -240,11 +207,10 @@ class NodeCommands(db.VolumeCommands, Commands): return result - @db.document_command(method='GET', cmd='feed', + @route('GET', ['context', None], cmd='feed', mime_type='application/json') - def feed(self, document, guid, layer, distro, request): - enforce(document == 'context') - context = self.volume['context'].get(guid) + def feed(self, request, layer, distro): + context = self.volume['context'].get(request.guid) implementations = self.volume['implementation'] versions = [] @@ -278,75 +244,122 @@ class NodeCommands(db.VolumeCommands, Commands): return result - def validate(self, *args): - return auth.try_validate(*args) + @route('GET', ['user', None], cmd='stats-info', + mime_type='application/json', acl=ACL.AUTH) + def user_stats_info(self, request): + status = {} + for rdb in stats_user.get_rrd(request.guid): + status[rdb.name] = rdb.last + stats_user.stats_user_step.value + + # TODO Process client configuration in more general manner + return {'enable': True, + 'step': stats_user.stats_user_step.value, + 'rras': ['RRA:AVERAGE:0.5:1:4320', 'RRA:AVERAGE:0.5:5:2016'], + 'status': status, + } - def call(self, request, response=None): - if node.static_url.value: - request.static_prefix = node.static_url.value - try: - result = db.VolumeCommands.call(self, request, response) - except http.StatusPass: - if self._stats is not None: - self._stats.log(request) - raise + @route('POST', ['user', None], cmd='stats-upload', acl=ACL.AUTH) + def user_stats_upload(self, request): + name = request.content['name'] + values = request.content['values'] + rrd = stats_user.get_rrd(request.guid) + for timestamp, values in values: + rrd[name].put(values, timestamp) + + @route('GET', ['report', None], cmd='log', mime_type='text/html') + def log(self, request): + # In further implementations, `data` might be a tarball + data = self.volume[request.resource].get(request.guid).meta('data') + if data and 'blob' in data: + return file(data['blob'], 'rb') else: - if self._stats is not None: - self._stats.log(request) - return result - - def resolve(self, request): - cmd = db.VolumeCommands.resolve(self, request) - if cmd is None: - return - - if cmd.permissions & db.ACCESS_AUTH: - enforce(self.validate(request, 'user'), http.Unauthorized, + return '' + + @preroute + def preroute(self, op, request): + user = request.environ.get('HTTP_X_SN_LOGIN') + if user and user not in self._authenticated and \ + (request.path != ['user'] or request.method != 'POST'): + _logger.debug('Logging %r user', user) + enforce(self.volume['user'].exists(user), http.Unauthorized, + 'Principal does not exist') + # TODO Process X-SN-signature + self._authenticated.add(user) + request.principal = user + + if op.acl & ACL.AUTH: + enforce(self.authorize(user, 'user'), http.Unauthorized, 'User is not authenticated') - - if cmd.permissions & db.ACCESS_AUTHOR and 'guid' in request: - if request['document'] == 'user': - allowed = (request.principal == request['guid']) + if op.acl & ACL.AUTHOR and request.guid: + if request.resource == 'user': + allowed = (user == request.guid) else: - doc = self.volume[request['document']].get(request['guid']) - allowed = (request.principal in doc['author']) - enforce(allowed or self.validate(request, 'root'), + doc = self.volume[request.resource].get(request.guid) + allowed = (user in doc['author']) + enforce(allowed or self.authorize(user, 'root'), http.Forbidden, 'Operation is permitted only for authors') + if op.acl & ACL.SUPERUSER: + enforce(self.authorize(user, 'root'), http.Forbidden, + 'Operation is permitted only for superusers') - return cmd + @postroute + def postroute(self, request, response, result, exception): + if exception is None or isinstance(exception, http.StatusPass): + if self._stats is not None: + self._stats.log(request) def on_create(self, request, props, event): - if request['document'] == 'user': + if request.resource == 'user': props['guid'], props['pubkey'] = _load_pubkey(props['pubkey']) - db.VolumeCommands.on_create(self, request, props, event) + db.Routes.on_create(self, request, props, event) def on_update(self, request, props, event): - db.VolumeCommands.on_update(self, request, props, event) + db.Routes.on_update(self, request, props, event) if 'deleted' in props.get('layer', []): event['event'] = 'delete' - @db.directory_command_pre(method='GET') - def _NodeCommands_find_pre(self, request): - if 'limit' not in request: + def find(self, request, reply): + limit = request.get('limit') + if limit is None or limit < 0: request['limit'] = node.find_limit.value - elif request['limit'] > node.find_limit.value: + elif limit > node.find_limit.value: _logger.warning('The find limit is restricted to %s', node.find_limit.value) request['limit'] = node.find_limit.value - layer = request.get('layer', ['public']) if 'deleted' in layer: _logger.warning('Requesting "deleted" layer') layer.remove('deleted') request['layer'] = layer + return db.Routes.find(self, request, reply) - @db.document_command_post(method='GET') - def _NodeCommands_get_post(self, request, response, result): - directory = self.volume[request['document']] - doc = directory.get(request['guid']) + def get(self, request, reply): + doc = self.volume[request.resource].get(request.guid) enforce('deleted' not in doc['layer'], http.NotFound, - 'Document deleted') - return result + 'Resource deleted') + return db.Routes.get(self, request, reply) + + def authorize(self, user, role): + if role == 'user' and user: + return True + + config_path = join(node.data_root.value, 'authorization.conf') + if exists(config_path): + mtime = os.stat(config_path).st_mtime + if mtime > self._auth_config_mtime: + self._auth_config_mtime = mtime + self._auth_config = ConfigParser() + self._auth_config.read(config_path) + if self._auth_config is None: + return False + + if not user: + user = 'anonymous' + if not self._auth_config.has_section(user): + user = 'DEFAULT' + if self._auth_config.has_option(user, role): + return self._auth_config.get(user, role).strip().lower() in \ + ('true', 'on', '1', 'allow') def _commit_stats(self): while True: @@ -354,21 +367,19 @@ class NodeCommands(db.VolumeCommands, Commands): self._stats.commit() def _clone(self, request): - enforce(request['document'] == 'context', 'No way to clone') - requires = {} - if 'requires' in request.query: + if 'requires' in request: for i in request['requires']: requires.update(parse_requires(i)) - request.query.pop('requires') + request.pop('requires') else: - request.query['limit'] = 1 + request['limit'] = 1 - if 'stability' not in request.query: - request.query['stability'] = 'stable' + if 'stability' not in request: + request['stability'] = 'stable' impls, __ = self.volume['implementation'].find( - context=request['guid'], order_by='-version', **request.query) + context=request.guid, order_by='-version', **request) impl = None for impl in impls: if requires: @@ -384,15 +395,15 @@ class NodeCommands(db.VolumeCommands, Commands): def _load_pubkey(pubkey): pubkey = pubkey.strip() try: - with util.NamedTemporaryFile() as key_file: + with toolkit.NamedTemporaryFile() as key_file: key_file.file.write(pubkey) key_file.file.flush() # SSH key needs to be converted to PKCS8 to ket M2Crypto read it - pubkey_pkcs8 = util.assert_call( + pubkey_pkcs8 = toolkit.assert_call( ['ssh-keygen', '-f', key_file.name, '-e', '-m', 'PKCS8']) except Exception: message = 'Cannot read DSS public key gotten for registeration' - exception(message) + toolkit.exception(message) if node.trust_users.value: logging.warning('Failed to read registration pubkey, ' 'but we trust users') diff --git a/sugar_network/node/slave.py b/sugar_network/node/slave.py index 2accd9e..5ea4140 100644 --- a/sugar_network/node/slave.py +++ b/sugar_network/node/slave.py @@ -21,42 +21,41 @@ from urlparse import urlsplit from os.path import join, dirname, exists, isabs from gettext import gettext as _ -from sugar_network import db, node, toolkit +from sugar_network import node, toolkit from sugar_network.client import api_url from sugar_network.node import sync, stats_user, files, volume -from sugar_network.node.commands import NodeCommands -from sugar_network.toolkit import util, http -from sugar_network.toolkit import exception, enforce +from sugar_network.node.routes import NodeRoutes +from sugar_network.toolkit.router import route, ACL +from sugar_network.toolkit import http, enforce _logger = logging.getLogger('node.slave') -class SlaveCommands(NodeCommands): +class SlaveRoutes(NodeRoutes): def __init__(self, key_path, volume_): - guid = util.ensure_key(key_path) - NodeCommands.__init__(self, guid, volume_) + guid = toolkit.ensure_key(key_path) + NodeRoutes.__init__(self, guid, volume_) self._key_path = key_path - self._push_seq = util.PersistentSequence( + self._push_seq = toolkit.PersistentSequence( join(volume_.root, 'push.sequence'), [1, None]) - self._pull_seq = util.PersistentSequence( + self._pull_seq = toolkit.PersistentSequence( join(volume_.root, 'pull.sequence'), [1, None]) - self._files_seq = util.PersistentSequence( + self._files_seq = toolkit.PersistentSequence( join(volume_.root, 'files.sequence'), [1, None]) self._master_guid = urlsplit(api_url.value).netloc self._offline_session = None - @db.volume_command(method='POST', cmd='online-sync', - permissions=db.ACCESS_LOCAL) + @route('POST', cmd='online-sync', acl=ACL.LOCAL) def online_sync(self, no_pull=False): profile = { 'name': self.guid, 'color': '#000000,#000000', 'machine_sn': '', 'machine_uuid': '', - 'pubkey': util.pubkey(self._key_path), + 'pubkey': toolkit.pubkey(self._key_path), } conn = http.Client(api_url.value, creds=(self.guid, self._key_path, lambda: profile)) @@ -82,8 +81,7 @@ class SlaveCommands(NodeCommands): headers={'Transfer-Encoding': 'chunked'}) self._import(sync.decode(response.raw), None) - @db.volume_command(method='POST', cmd='offline-sync', - permissions=db.ACCESS_LOCAL) + @route('POST', cmd='offline-sync', acl=ACL.LOCAL) def offline_sync(self, path): enforce(node.sync_layers.value and 'public' not in node.sync_layers.value, @@ -101,7 +99,7 @@ class SlaveCommands(NodeCommands): self._offline_session = self._offline_sync(path, **(self._offline_session or {})) except Exception: - exception(_logger, 'Failed to complete synchronization') + toolkit.exception(_logger, 'Failed to complete synchronization') self._offline_session = None raise @@ -115,7 +113,7 @@ class SlaveCommands(NodeCommands): push = [] if push_seq is None: - push_seq = util.Sequence(self._push_seq) + push_seq = toolkit.Sequence(self._push_seq) if stats_seq is None: stats_seq = {} if session is None: @@ -141,7 +139,7 @@ class SlaveCommands(NodeCommands): 'progress': _('Generating new sneakernet package'), }) - diff_seq = util.Sequence([]) + diff_seq = toolkit.Sequence([]) push.append(('diff', None, volume.diff(self.volume, push_seq, diff_seq))) if stats_user.stats_user.value: diff --git a/sugar_network/node/stats_node.py b/sugar_network/node/stats_node.py index 3b71474..a2a2827 100644 --- a/sugar_network/node/stats_node.py +++ b/sugar_network/node/stats_node.py @@ -56,22 +56,22 @@ class Sniffer(object): for cls in (_UserStats, _ContextStats, _ImplementationStats, _ReportStats, _ReviewStats, _FeedbackStats, _SolutionStats, _ArtifactStats, _CommentStats): - self._stats[cls.DOCUMENT] = cls(self._stats, volume) + self._stats[cls.RESOURCE] = cls(self._stats, volume) def log(self, request): - if 'cmd' in request: + if request.cmd: return - stats = self._stats.get(request.get('document')) + stats = self._stats.get(request.resource) if stats is not None: stats.log(request) def commit(self, timestamp=None): _logger.heartbeat('Commit node stats') - for document, stats in self._stats.items(): + for resource, stats in self._stats.items(): values = stats.commit() if values is not None: - self.rrd[document].put(values, timestamp=timestamp) + self.rrd[resource].put(values, timestamp=timestamp) class _ObjectStats(object): @@ -82,7 +82,7 @@ class _ObjectStats(object): class _Stats(object): - DOCUMENT = None + RESOURCE = None OWNERS = [] active = None @@ -90,6 +90,7 @@ class _Stats(object): def __init__(self, stats, volume): self._stats = stats self._volume = volume + self._directory = volume[self.RESOURCE] def log(self, request): context = None @@ -104,30 +105,26 @@ class _Stats(object): else: return self._volume[owner].get(guid)['context'] - method = request['method'] - if method == 'GET': - if 'guid' in request: - if self.DOCUMENT == 'context': - context = request['guid'] - elif self.DOCUMENT != 'user': - doc = self._volume[self.DOCUMENT].get(request['guid']) - context = doc['context'] - else: + if request.method == 'GET': + if not request.guid: context = parse_context(request) - elif method == 'PUT': - guid = request['guid'] - if self.DOCUMENT == 'context': - context = guid + elif self.RESOURCE == 'context': + context = request.guid + elif self.RESOURCE != 'user': + context = self._directory.get(request.guid)['context'] + elif request.method == 'PUT': + if self.RESOURCE == 'context': + context = request.guid else: context = request.content.get('context') if not context: - context = self._volume[self.DOCUMENT].get(guid)['context'] - elif method == 'POST': + context = self._directory.get(request.guid)['context'] + elif request.method == 'POST': context = parse_context(request.content) if request.principal: stats = self._stats['user'] - if method in ('POST', 'PUT', 'DELETE'): + if request.method in ('POST', 'PUT', 'DELETE'): stats.effective.add(request.principal) stats.active.add(request.principal) @@ -154,21 +151,20 @@ class _ResourceStats(_Stats): def __init__(self, stats, volume): _Stats.__init__(self, stats, volume) - self.total = volume[self.DOCUMENT].find(limit=0)[1] + self.total = volume[self.RESOURCE].find(limit=0)[1] def log(self, request): result = _Stats.log(self, request) - method = request['method'] - if method == 'GET': - if 'guid' in request and 'prop' not in request: + if request.method == 'GET': + if request.guid and not request.prop: self.viewed += 1 - elif method == 'PUT': + elif request.method == 'PUT': self.updated += 1 - elif method == 'POST': + elif request.method == 'POST': self.total += 1 self.created += 1 - elif method == 'DELETE': + elif request.method == 'DELETE': self.total -= 1 self.deleted += 1 @@ -176,14 +172,13 @@ class _ResourceStats(_Stats): def commit(self): if type(self.active) is dict: - directory = self._volume[self.DOCUMENT] for guid, stats in self.active.items(): if not stats.reviews: continue - reviews, rating = directory.get(guid)['reviews'] + reviews, rating = self._directory.get(guid)['reviews'] reviews += stats.reviews rating += stats.rating - directory.update(guid, { + self._directory.update(guid, { 'reviews': [reviews, rating], 'rating': int(round(float(rating) / reviews)), }) @@ -208,7 +203,7 @@ class _ResourceStats(_Stats): class _UserStats(_ResourceStats): - DOCUMENT = 'user' + RESOURCE = 'user' def __init__(self, stats, volume): _ResourceStats.__init__(self, stats, volume) @@ -224,7 +219,7 @@ class _UserStats(_ResourceStats): class _ContextStats(_ResourceStats): - DOCUMENT = 'context' + RESOURCE = 'context' released = 0 failed = 0 @@ -247,35 +242,34 @@ class _ContextStats(_ResourceStats): class _ImplementationStats(_Stats): - DOCUMENT = 'implementation' + RESOURCE = 'implementation' OWNERS = ['context'] def log(self, request): _Stats.log(self, request) - method = request['method'] - if method == 'GET': - if request.get('prop') == 'data': + if request.method == 'GET': + if request.prop == 'data': self._stats['context'].downloaded += 1 - elif method == 'POST': + elif request.method == 'POST': self._stats['context'].released += 1 class _ReportStats(_Stats): - DOCUMENT = 'report' + RESOURCE = 'report' OWNERS = ['context', 'implementation'] def log(self, request): _Stats.log(self, request) - if request['method'] == 'POST': + if request.method == 'POST': self._stats['context'].failed += 1 class _ReviewStats(_ResourceStats): - DOCUMENT = 'review' + RESOURCE = 'review' OWNERS = ['artifact', 'context'] commented = 0 @@ -283,7 +277,7 @@ class _ReviewStats(_ResourceStats): def log(self, request): context = _ResourceStats.log(self, request) - if request['method'] == 'POST': + if request.method == 'POST': if request.content.get('artifact'): artifact = self._stats['artifact'] stats = artifact.active_object(request.content['artifact']) @@ -302,7 +296,7 @@ class _ReviewStats(_ResourceStats): class _FeedbackStats(_ResourceStats): - DOCUMENT = 'feedback' + RESOURCE = 'feedback' OWNERS = ['context'] solutions = 0 @@ -320,7 +314,7 @@ class _FeedbackStats(_ResourceStats): def log(self, request): _ResourceStats.log(self, request) - if request['method'] in ('POST', 'PUT'): + if request.method in ('POST', 'PUT'): if 'solution' in request.content: if request.content['solution'] is None: self.rejected += 1 @@ -339,7 +333,7 @@ class _FeedbackStats(_ResourceStats): class _SolutionStats(_ResourceStats): - DOCUMENT = 'solution' + RESOURCE = 'solution' OWNERS = ['feedback'] commented = 0 @@ -352,7 +346,7 @@ class _SolutionStats(_ResourceStats): class _ArtifactStats(_ResourceStats): - DOCUMENT = 'artifact' + RESOURCE = 'artifact' OWNERS = ['context'] reviewed = 0 @@ -365,8 +359,8 @@ class _ArtifactStats(_ResourceStats): def log(self, request): _ResourceStats.log(self, request) - if request['method'] == 'GET': - if request.get('prop') == 'data': + if request.method == 'GET': + if request.prop == 'data': self.downloaded += 1 def commit(self): @@ -379,13 +373,13 @@ class _ArtifactStats(_ResourceStats): class _CommentStats(_Stats): - DOCUMENT = 'comment' + RESOURCE = 'comment' OWNERS = ['solution', 'feedback', 'review'] def log(self, request): _Stats.log(self, request) - if request['method'] == 'POST': + if request.method == 'POST': for owner in ('solution', 'feedback', 'review'): if request.content.get(owner): self._stats[owner].commented += 1 diff --git a/sugar_network/node/stats_user.py b/sugar_network/node/stats_user.py index 686ad52..9265502 100644 --- a/sugar_network/node/stats_user.py +++ b/sugar_network/node/stats_user.py @@ -17,9 +17,9 @@ import os import logging from os.path import join, exists, isdir -from sugar_network import node +from sugar_network import node, toolkit from sugar_network.toolkit.rrd import Rrd -from sugar_network.toolkit import Option, util, pylru, enforce +from sugar_network.toolkit import Option, pylru, enforce stats_user = Option( @@ -66,11 +66,13 @@ def diff(in_info=None): in_seq = in_info[user].get(db.name) if in_seq is None: - in_seq = in_info[user][db.name] = util.PersistentSequence( + in_seq = toolkit.PersistentSequence( join(rrd.root, db.name + '.push'), [1, None]) - elif in_seq is not util.Sequence: - in_seq = in_info[user][db.name] = util.Sequence(in_seq) - out_seq = out_info[user].setdefault(db.name, util.Sequence()) + in_info[user][db.name] = in_seq + elif in_seq is not toolkit.Sequence: + in_seq = in_info[user][db.name] = toolkit.Sequence(in_seq) + out_seq = out_info[user].setdefault(db.name, + toolkit.Sequence()) for start, end in in_seq: for timestamp, values in \ @@ -104,7 +106,7 @@ def merge(packet): def commit(info): for user, dbs in info.items(): for db_name, merged in dbs.items(): - seq = util.PersistentSequence( + seq = toolkit.PersistentSequence( _rrd_path(user, db_name + '.push'), [1, None]) seq.exclude(merged) seq.commit() diff --git a/sugar_network/node/sync.py b/sugar_network/node/sync.py index e95f989..b0a20bf 100644 --- a/sugar_network/node/sync.py +++ b/sugar_network/node/sync.py @@ -23,7 +23,7 @@ from types import GeneratorType from os.path import exists, join, dirname, basename, splitext from sugar_network import toolkit -from sugar_network.toolkit import coroutine, util, BUFFER_SIZE, enforce +from sugar_network.toolkit import coroutine, enforce # Filename suffix to use for sneakernet synchronization files @@ -109,7 +109,7 @@ def sneakernet_encode(packets, root=None, limit=None, path=None, **header): if not exists(root): os.makedirs(root) filename = toolkit.uuid() + _SNEAKERNET_SUFFIX - path = util.unique_filename(root, filename) + path = toolkit.unique_filename(root, filename) else: filename = splitext(basename(path))[0] + _SNEAKERNET_SUFFIX if 'filename' not in header: @@ -212,7 +212,7 @@ class _PacketsIterator(object): def __init__(self, stream): if not hasattr(stream, 'readline'): - stream.readline = lambda: util.readline(stream) + stream.readline = lambda: toolkit.readline(stream) if hasattr(stream, 'seek'): self._seek = stream.seek self._stream = stream @@ -268,7 +268,7 @@ class _PacketsIterator(object): # pylint: disable-msg=E0202 def _seek(self, distance, where): while distance: - chunk = self._stream.read(min(distance, BUFFER_SIZE)) + chunk = self._stream.read(min(distance, toolkit.BUFFER_SIZE)) distance -= len(chunk) @@ -278,7 +278,7 @@ class _Blob(object): self._stream = stream self.size_to_read = size - def read(self, size=BUFFER_SIZE): + def read(self, size=toolkit.BUFFER_SIZE): chunk = self._stream.read(min(size, self.size_to_read)) self.size_to_read -= len(chunk) return chunk @@ -304,4 +304,4 @@ class _GzipStream(object): self._buffer += self._zip.decompress(chunk) def readline(self): - return util.readline(self) + return toolkit.readline(self) diff --git a/sugar_network/node/volume.py b/sugar_network/node/volume.py index 5a6d0f6..a31a28b 100644 --- a/sugar_network/node/volume.py +++ b/sugar_network/node/volume.py @@ -15,7 +15,9 @@ import logging -from sugar_network.toolkit import BUFFER_SIZE, http, util, coroutine, enforce +from sugar_network import toolkit +from sugar_network.toolkit.router import Request +from sugar_network.toolkit import http, coroutine, enforce # Apply node level layer for these documents @@ -28,7 +30,7 @@ def diff(volume, in_seq, out_seq=None, exclude_seq=None, layer=None, fetch_blobs=False, ignore_documents=None, **kwargs): connection = http.Client() if out_seq is None: - out_seq = util.Sequence([]) + out_seq = toolkit.Sequence([]) is_the_only_seq = not out_seq if layer: if isinstance(layer, basestring): @@ -40,18 +42,18 @@ def diff(volume, in_seq, out_seq=None, exclude_seq=None, layer=None, continue coroutine.dispatch() directory.commit() - yield {'document': document} + yield {'resource': document} for guid, patch in directory.diff(in_seq, exclude_seq, layer=layer if document in _LIMITED_RESOURCES else None): adiff = {} - adiff_seq = util.Sequence() + adiff_seq = toolkit.Sequence() for prop, meta, seqno in patch: if 'blob' in meta: blob_path = meta.pop('blob') yield {'guid': guid, 'diff': {prop: meta}, 'blob_size': meta['blob_size'], - 'blob': util.iter_file(blob_path), + 'blob': toolkit.iter_file(blob_path), } elif fetch_blobs and 'url' in meta: url = meta.pop('url') @@ -69,7 +71,7 @@ def diff(volume, in_seq, out_seq=None, exclude_seq=None, layer=None, 'diff': {prop: meta}, 'blob_size': int(blob.headers['Content-Length']), - 'blob': blob.iter_content(BUFFER_SIZE), + 'blob': blob.iter_content(toolkit.BUFFER_SIZE), } else: adiff[prop] = meta @@ -89,12 +91,12 @@ def diff(volume, in_seq, out_seq=None, exclude_seq=None, layer=None, def merge(volume, records, shift_seqno=True, node_stats=None): document = None directory = None - commit_seq = util.Sequence() - merged_seq = util.Sequence() + commit_seq = toolkit.Sequence() + merged_seq = toolkit.Sequence() synced = False for record in records: - document_ = record.get('document') + document_ = record.get('resource') if document_: document = document_ directory = volume[document_] @@ -109,9 +111,7 @@ def merge(volume, records, shift_seqno=True, node_stats=None): merged_seq.include(seqno, seqno) if node_stats is not None and document == 'review': - request = _Request() - request['document'] = document - request['method'] = 'POST' + request = Request(method='POST', path=[document]) patch = record['diff'] request.content = { 'context': patch['context']['value'], @@ -128,12 +128,6 @@ def merge(volume, records, shift_seqno=True, node_stats=None): continue if synced: - volume.notify({'event': 'sync'}) + volume.broadcast({'event': 'sync'}) return commit_seq, merged_seq - - -class _Request(dict): - - principal = None - content = None diff --git a/sugar_network/resources/user.py b/sugar_network/resources/user.py deleted file mode 100644 index 8aaca09..0000000 --- a/sugar_network/resources/user.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright (C) 2012-2013 Aleksey Lim -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -from sugar_network import db -from sugar_network.node import stats_user -from sugar_network.toolkit import http, enforce - - -class User(db.Document): - - @db.indexed_property(prefix='L', typecast=[], default=['public']) - def layer(self, value): - return value - - @db.indexed_property(slot=1, prefix='N', full_text=True) - def name(self, value): - return value - - @db.stored_property() - def color(self, value): - return value - - @db.indexed_property(prefix='S', default='', - permissions=db.ACCESS_CREATE | db.ACCESS_WRITE) - def machine_sn(self, value): - return value - - @db.indexed_property(prefix='U', default='', - permissions=db.ACCESS_CREATE | db.ACCESS_WRITE) - def machine_uuid(self, value): - return value - - @db.stored_property(permissions=db.ACCESS_CREATE) - def pubkey(self, value): - return value - - @db.indexed_property(prefix='T', full_text=True, default=[], typecast=[]) - def tags(self, value): - return value - - @db.indexed_property(prefix='P', full_text=True, default='') - def location(self, value): - return value - - @db.indexed_property(slot=2, prefix='B', default=0, typecast=int) - def birthday(self, value): - return value - - @db.document_command(method='GET', cmd='stats-info', - mime_type='application/json') - def _stats_info(self, request): - enforce(request.principal == self['guid'], http.Forbidden, - 'Operation is permitted only for authors') - - status = {} - for rdb in stats_user.get_rrd(self.guid): - status[rdb.name] = rdb.last + stats_user.stats_user_step.value - - # TODO Process client configuration in more general manner - return {'enable': True, - 'step': stats_user.stats_user_step.value, - 'rras': ['RRA:AVERAGE:0.5:1:4320', 'RRA:AVERAGE:0.5:5:2016'], - 'status': status, - } - - @db.document_command(method='POST', cmd='stats-upload') - def _stats_upload(self, request): - enforce(request.principal == self['guid'], http.Forbidden, - 'Operation is permitted only for authors') - - name = request.content['name'] - values = request.content['values'] - rrd = stats_user.get_rrd(self.guid) - for timestamp, values in values: - rrd[name].put(values, timestamp) diff --git a/sugar_network/resources/volume.py b/sugar_network/resources/volume.py deleted file mode 100644 index feb23ff..0000000 --- a/sugar_network/resources/volume.py +++ /dev/null @@ -1,243 +0,0 @@ -# Copyright (C) 2012-2013 Aleksey Lim -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -import json -import logging - -from sugar_network import db -from sugar_network.toolkit import coroutine, enforce - - -AUTHOR_INSYSTEM = 1 -AUTHOR_ORIGINAL = 2 -AUTHOR_ALL = (AUTHOR_INSYSTEM | AUTHOR_ORIGINAL) - - -_logger = logging.getLogger('resources.volume') - - -def _reprcast_authors(value): - if isinstance(value, dict): - for guid, props in value.items(): - if not isinstance(props, dict): - yield guid - else: - if 'name' in props: - yield props['name'] - if not (props['role'] & AUTHOR_INSYSTEM): - yield guid - else: - yield value - - -class Resource(db.Document): - - @db.indexed_property(prefix='RA', typecast=dict, full_text=True, - default={}, reprcast=_reprcast_authors, permissions=db.ACCESS_READ) - def author(self, value): - result = [] - for guid, props in sorted(value.items(), - cmp=lambda x, y: cmp(x[1]['order'], y[1]['order'])): - if 'name' in props: - result.append({ - 'guid': guid, - 'name': props['name'], - 'role': props['role'], - }) - else: - result.append({ - 'name': guid, - 'role': props['role'], - }) - return result - - @author.setter - def author(self, value): - if not self.request.principal: - return {} - return self._useradd(self.request.principal, AUTHOR_ORIGINAL) - - @db.document_command(method='PUT', cmd='useradd', - arguments={'role': db.to_int}, - permissions=db.ACCESS_AUTH | db.ACCESS_AUTHOR) - def useradd(self, user, role): - enforce(user, "Argument 'user' is not specified") - self.directory.update(self.guid, {'author': self._useradd(user, role)}) - - @db.document_command(method='PUT', cmd='userdel', - permissions=db.ACCESS_AUTH | db.ACCESS_AUTHOR) - def userdel(self, user): - enforce(user, "Argument 'user' is not specified") - enforce(user != self.request.principal, 'Cannot remove yourself') - author = self['author'] - enforce(user in author, 'No such user') - del author[user] - self.directory.update(self.guid, {'author': author}) - - @db.indexed_property(prefix='RL', typecast=[], default=['public']) - def layer(self, value): - return value - - @db.indexed_property(prefix='RT', full_text=True, default=[], typecast=[]) - def tags(self, value): - return value - - def _useradd(self, user, role): - props = {} - - if role is None: - role = 0 - users = self.volume['user'] - if users.exists(user): - props['name'] = users.get(user)['name'] - role |= AUTHOR_INSYSTEM - else: - role &= ~AUTHOR_INSYSTEM - props['role'] = role & AUTHOR_ALL - - author = self['author'] or {} - if user in author: - author[user].update(props) - else: - if author: - order = max(author.values(), key=lambda x: x['order'])['order'] - props['order'] = order + 1 - else: - props['order'] = 0 - author[user] = props - - return author - - -class Volume(db.Volume): - - RESOURCES = ( - 'sugar_network.resources.artifact', - 'sugar_network.resources.comment', - 'sugar_network.resources.context', - 'sugar_network.resources.implementation', - 'sugar_network.resources.notification', - 'sugar_network.resources.feedback', - 'sugar_network.resources.report', - 'sugar_network.resources.review', - 'sugar_network.resources.solution', - 'sugar_network.resources.user', - ) - - def __init__(self, root, document_classes=None): - if document_classes is None: - document_classes = Volume.RESOURCES - self._populators = coroutine.Pool() - db.Volume.__init__(self, root, document_classes) - - def close(self): - self._populators.kill() - db.Volume.close(self) - - def _open(self, name, document): - directory = db.Volume._open(self, name, document) - self._populators.spawn(self._populate, directory) - return directory - - def _populate(self, directory): - for __ in directory.populate(): - coroutine.dispatch() - - -class Commands(object): - - def __init__(self): - self._pooler = _Pooler() - - @db.volume_command(method='GET', cmd='subscribe', - mime_type='text/event-stream') - def subscribe(self, request=None, response=None, ping=False, **condition): - """Subscribe to Server-Sent Events.""" - if request is not None and not condition: - condition = request.query - if response is not None: - response.content_type = 'text/event-stream' - response['Cache-Control'] = 'no-cache' - peer = 'anonymous' - if hasattr(request, 'environ'): - peer = request.environ.get('HTTP_SUGAR_USER') or peer - return self._pull_events(peer, ping, condition) - - @db.volume_command(method='POST', cmd='broadcast', - mime_type='application/json', permissions=db.ACCESS_LOCAL) - def broadcast(self, event=None, request=None): - if request is not None: - event = request.content - _logger.debug('Publish event: %r', event) - self._pooler.notify_all(event) - coroutine.dispatch() - - def _pull_events(self, peer, ping, condition): - _logger.debug('Start pulling events to %s user', peer) - - if ping: - # XXX The whole commands' kwargs handling should be redesigned - if 'ping' in condition: - condition.pop('ping') - # If non-greenlet application needs only to initiate - # a subscription and do not stuck in waiting for the first event, - # it should pass `ping` argument to return fake event to unblock - # `GET /?cmd=subscribe` call. - yield 'data: %s\n\n' % json.dumps({'event': 'pong'}) - - try: - while True: - event = self._pooler.wait() - for key, value in condition.items(): - if value.startswith('!'): - if event.get(key) == value[1:]: - break - elif event.get(key) != value: - break - else: - yield 'data: %s\n\n' % json.dumps(event) - finally: - _logger.debug('Stop pulling events to %s user', peer) - - -class _Pooler(object): - """One-producer-to-many-consumers events delivery.""" - - def __init__(self): - self._value = None - self._waiters = 0 - self._ready = coroutine.Event() - self._open = coroutine.Event() - self._open.set() - - def wait(self): - self._open.wait() - self._waiters += 1 - try: - self._ready.wait() - finally: - self._waiters -= 1 - if self._waiters == 0: - self._ready.clear() - self._open.set() - return self._value - - def notify_all(self, value=None): - self._open.wait() - if not self._waiters: - return - self._open.clear() - self._value = value - self._ready.set() diff --git a/sugar_network/toolkit/__init__.py b/sugar_network/toolkit/__init__.py index c797613..e753a78 100644 --- a/sugar_network/toolkit/__init__.py +++ b/sugar_network/toolkit/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Aleksey Lim +# Copyright (C) 2011-2013 Aleksey Lim # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -13,9 +13,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import os import sys +import json +import errno import logging -from os.path import join +import tempfile +import collections +from os.path import exists, join, islink, isdir, dirname, basename, abspath +from os.path import lexists, isfile from sugar_network.toolkit.options import Option @@ -28,6 +34,8 @@ cachedir = Option( 'might take considerable number of bytes', default='/var/cache/sugar-network', name='cachedir') +_logger = logging.getLogger('toolkit') + def enforce(condition, error=None, *args): """Make an assertion in runtime. @@ -128,6 +136,42 @@ def default_lang(): return _default_lang +def gettext(value, accept_language=None): + if not value: + return '' + if not isinstance(value, dict): + return value + + if accept_language is None: + accept_language = [default_lang()] + elif isinstance(accept_language, basestring): + accept_language = [accept_language] + accept_language.append('en') + + stripped_value = None + for lang in accept_language: + result = value.get(lang) + if result is not None: + return result + + prime_lang = lang.split('-')[0] + if prime_lang != lang: + result = value.get(prime_lang) + if result is not None: + return result + + if stripped_value is None: + stripped_value = {} + for k, v in value.items(): + if '-' in k: + stripped_value[k.split('-', 1)[0]] = v + result = stripped_value.get(prime_lang) + if result is not None: + return result + + return value[min(value.keys())] + + def uuid(): """Generate GUID value. @@ -143,4 +187,655 @@ def uuid(): return ''.join(str(uuid1()).split('-')) +def init_logging(debug_level): + # pylint: disable-msg=W0212 + + logging.addLevelName(9, 'TRACE') + logging.addLevelName(8, 'HEARTBEAT') + + logging.Logger.trace = lambda self, message, *args, **kwargs: None + logging.Logger.heartbeat = lambda self, message, *args, **kwargs: None + + if debug_level < 3: + _disable_logger([ + 'requests.packages.urllib3.connectionpool', + 'requests.packages.urllib3.poolmanager', + 'requests.packages.urllib3.response', + 'requests.packages.urllib3', + 'inotify', + 'netlink', + 'sugar_stats', + '0install', + ]) + elif debug_level < 4: + logging.Logger.trace = lambda self, message, *args, **kwargs: \ + self._log(9, message, args, **kwargs) + _disable_logger(['sugar_stats']) + else: + logging.Logger.trace = lambda self, message, *args, **kwargs: \ + self._log(9, message, args, **kwargs) + logging.Logger.heartbeat = lambda self, message, *args, **kwargs: \ + self._log(8, message, args, **kwargs) + + +def ensure_key(path): + import hashlib + if not exists(path): + if 'SSH_ASKPASS' in os.environ: + # Otherwise ssh-keygen will popup auth dialogs on registeration + del os.environ['SSH_ASKPASS'] + if not exists(dirname(path)): + os.makedirs(dirname(path)) + _logger.info('Create DSA key') + assert_call([ + '/usr/bin/ssh-keygen', '-q', '-t', 'dsa', '-f', path, + '-C', '', '-N', '']) + key = pubkey(path).split()[1] + return str(hashlib.sha1(key).hexdigest()) + + +def pubkey(path): + with file(path + '.pub') as f: + for line in f.readlines(): + line = line.strip() + if line.startswith('ssh-'): + return line + raise RuntimeError('No valid DSA public keys in %r' % path) + + +def iter_file(*path): + with file(join(*path), 'rb') as f: + while True: + chunk = f.read(BUFFER_SIZE) + if not chunk: + return + yield chunk + + +def readline(stream, limit=None): + line = bytearray() + while limit is None or len(line) < limit: + char = stream.read(1) + if not char: + break + line.append(char) + if char == '\n': + break + return bytes(line) + + +def default_route_exists(): + with file('/proc/self/net/route') as f: + # Skip header + f.readline() + while True: + line = f.readline() + if not line: + break + if int(line.split('\t', 2)[1], 16) == 0: + return True + + +def spawn(cmd_filename, *args): + _logger.trace('Spawn %s%r', cmd_filename, args) + + if os.fork(): + return + + os.execvp(cmd_filename, (cmd_filename,) + args) + + +def symlink(src, dst): + if not isfile(src): + _logger.debug('Cannot link %r to %r, source file is absent', src, dst) + return + + _logger.trace('Link %r to %r', src, dst) + + if lexists(dst): + os.unlink(dst) + elif not exists(dirname(dst)): + os.makedirs(dirname(dst)) + os.symlink(src, dst) + + +def svg_to_png(src_path, dst_path, width, height): + import rsvg + import cairo + + svg = rsvg.Handle(src_path) + + surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, width, height) + context = cairo.Context(surface) + scale = min( + float(width) / svg.props.width, + float(height) / svg.props.height) + context.scale(scale, scale) + svg.render_cairo(context) + + surface.write_to_png(dst_path) + + +def assert_call(cmd, stdin=None, **kwargs): + """Variant of `call` method with raising exception of errors. + + :param cmd: + commad to execute, might be string or argv list + :param stdin: + text that will be used as an input for executed process + + """ + return call(cmd, stdin=stdin, asserts=True, **kwargs) + + +def call(cmd, stdin=None, asserts=False, raw=False, error_cb=None, **kwargs): + """Convenient wrapper around subprocess call. + + Note, this function is intended for processes that output finite + and not big amount of text. + + :param cmd: + commad to execute, might be string or argv list + :param stdin: + text that will be used as an input for executed process + :param asserts: + whether to raise `RuntimeError` on fail execution status + :param error_cb: + call callback(stderr) on getting error exit status from the process + :returns: + `None` on errors, otherwise `str` value of stdout + + """ + import subprocess + + stdout, stderr = None, None + returncode = 1 + try: + logging.debug('Exec %r', cmd) + process = subprocess.Popen(cmd, stderr=subprocess.PIPE, + stdout=subprocess.PIPE, stdin=subprocess.PIPE, **kwargs) + if stdin is not None: + process.stdin.write(stdin) + process.stdin.close() + # Avoid using Popen.communicate() + # http://bugs.python.org/issue4216#msg77582 + process.wait() + stdout = _nb_read(process.stdout) + stderr = _nb_read(process.stderr) + if not raw: + stdout = stdout.strip() + stderr = stderr.strip() + returncode = process.returncode + enforce(returncode == 0, 'Exit status is an error') + logging.debug('Successfully executed stdout=%r stderr=%r', + stdout.split('\n'), stderr.split('\n')) + return stdout + except Exception, error: + logging.debug('Failed to execute error="%s" stdout=%r stderr=%r', + error, str(stdout).split('\n'), str(stderr).split('\n')) + if asserts: + if type(cmd) not in (str, unicode): + cmd = ' '.join(cmd) + raise RuntimeError('Failed to execute "%s" command: %s' % + (cmd, error)) + elif error_cb is not None: + error_cb(returncode, stdout, stderr) + + +def cptree(src, dst): + """Efficient version of copying directories. + + Function will try to make hard links for copying files at first and + will fallback to regular copying overwise. + + :param src: + path to the source directory + :param dst: + path to the new directory + + """ + import shutil + + if abspath(src) == abspath(dst): + return + + do_copy = [] + src = abspath(src) + dst = abspath(dst) + + def link(src, dst): + if not exists(dirname(dst)): + os.makedirs(dirname(dst)) + + if islink(src): + link_to = os.readlink(src) + os.symlink(link_to, dst) + elif isdir(src): + cptree(src, dst) + elif do_copy: + # The first hard link was not set, do regular copying for the rest + shutil.copy(src, dst) + else: + if exists(dst) and os.stat(src).st_ino == os.stat(dst).st_ino: + return + if os.access(src, os.W_OK): + try: + os.link(src, dst) + except OSError: + do_copy.append(True) + shutil.copy(src, dst) + shutil.copystat(src, dst) + else: + # Avoid copystat from not current users + shutil.copy(src, dst) + + if isdir(src): + for root, __, files in os.walk(src): + dst_root = join(dst, root[len(src):].lstrip(os.sep)) + if not exists(dst_root): + os.makedirs(dst_root) + for i in files: + link(join(root, i), join(dst_root, i)) + else: + link(src, dst) + + +def new_file(path, mode=0644): + """Atomic new file creation. + + Method will create temporaty file in the same directory as the specified + one. When file object associated with this temporaty file will be closed, + temporaty file will be renamed to the final destination. + + :param path: + path to save final file to + :param mode: + mode for new file + :returns: + file object + + """ + result = _NewFile(dir=dirname(path), prefix=basename(path)) + result.dst_path = path + os.fchmod(result.fileno(), mode) + return result + + +def unique_filename(root, filename): + path = join(root, filename) + if exists(path): + name, suffix = os.path.splitext(filename) + for dup_num in xrange(1, 255): + path = join(root, name + '_' + str(dup_num) + suffix) + if not exists(path): + break + else: + raise RuntimeError('Cannot find unique filename for %r' % + join(root, filename)) + return path + + +def TemporaryFile(*args, **kwargs): + if cachedir.value: + if not exists(cachedir.value): + os.makedirs(cachedir.value) + kwargs['dir'] = cachedir.value + return tempfile.TemporaryFile(*args, **kwargs) + + +class NamedTemporaryFile(object): + + def __init__(self, *args, **kwargs): + if cachedir.value: + if not exists(cachedir.value): + os.makedirs(cachedir.value) + kwargs['dir'] = cachedir.value + self._file = tempfile.NamedTemporaryFile(*args, **kwargs) + + def close(self): + try: + self._file.close() + except OSError, error: + if error.errno != errno.ENOENT: + raise + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + def __getattr__(self, name): + return getattr(self._file, name) + + +class Seqno(object): + """Sequence number counter with persistent storing in a file.""" + + def __init__(self, path): + """ + :param path: + path to file to [re]store seqno value + + """ + self._path = path + self._value = 0 + + if exists(path): + with file(path) as f: + self._value = int(f.read().strip()) + + self._orig_value = self._value + + @property + def value(self): + """Current seqno value.""" + return self._value + + def next(self): + """Incerement seqno. + + :returns: + new seqno value + + """ + self._value += 1 + return self._value + + def commit(self): + """Store current seqno value in a file. + + :returns: + `True` if commit was happened + + """ + if self._value == self._orig_value: + return False + with new_file(self._path) as f: + f.write(str(self._value)) + f.flush() + os.fsync(f.fileno()) + self._orig_value = self._value + return True + + +class Sequence(list): + """List of sorted and non-overlapping ranges. + + List items are ranges, [`start`, `stop']. If `start` or `stop` + is `None`, it means the beginning or ending of the entire scale. + + """ + + def __init__(self, value=None, empty_value=None): + """ + :param value: + default value to initialize range + :param empty_value: + if not `None`, the initial value for empty range + + """ + if empty_value is None: + self._empty_value = [] + else: + self._empty_value = [empty_value] + + if value: + self.extend(value) + else: + self.clear() + + def __contains__(self, value): + for start, end in self: + if value >= start and (end is None or value <= end): + return True + else: + return False + + @property + def empty(self): + """Is timeline in the initial state.""" + return self == self._empty_value + + def clear(self): + """Reset range to the initial value.""" + self[:] = self._empty_value + + def stretch(self): + """Remove all holes between the first and the last items.""" + if self: + self[:] = [[self[0][0], self[-1][-1]]] + + def include(self, start, end=None): + """Include specified range. + + :param start: + either including range start or a list of + (`start`, `end`) pairs + :param end: + including range end + + """ + if issubclass(type(start), collections.Iterable): + for range_start, range_end in start: + self._include(range_start, range_end) + elif start is not None: + self._include(start, end) + + def exclude(self, start, end=None): + """Exclude specified range. + + :param start: + either excluding range start or a list of + (`start`, `end`) pairs + :param end: + excluding range end + + """ + if issubclass(type(start), collections.Iterable): + for range_start, range_end in start: + self._exclude(range_start, range_end) + else: + enforce(end is not None) + self._exclude(start, end) + + def _include(self, range_start, range_end): + if range_start is None: + range_start = 1 + + range_start_new = None + range_start_i = 0 + + for range_start_i, (start, end) in enumerate(self): + if range_end is not None and start - 1 > range_end: + break + if (range_end is None or start - 1 <= range_end) and \ + (end is None or end + 1 >= range_start): + range_start_new = min(start, range_start) + break + else: + range_start_i += 1 + + if range_start_new is None: + self.insert(range_start_i, [range_start, range_end]) + return + + range_end_new = range_end + range_end_i = range_start_i + for i, (start, end) in enumerate(self[range_start_i:]): + if range_end is not None and start - 1 > range_end: + break + if range_end is None or end is None: + range_end_new = None + else: + range_end_new = max(end, range_end) + range_end_i = range_start_i + i + + del self[range_start_i:range_end_i] + self[range_start_i] = [range_start_new, range_end_new] + + def _exclude(self, range_start, range_end): + if range_start is None: + range_start = 1 + enforce(range_end is not None) + enforce(range_start <= range_end and range_start > 0, + 'Start value %r is less than 0 or not less than %r', + range_start, range_end) + + for i, interval in enumerate(self): + start, end = interval + + if end is not None and end < range_start: + # Current `interval` is below new one + continue + + if range_end is not None and range_end < start: + # Current `interval` is above new one + continue + + if end is None or end > range_end: + # Current `interval` will exist after changing + self[i] = [range_end + 1, end] + if start < range_start: + self.insert(i, [start, range_start - 1]) + else: + if start < range_start: + self[i] = [start, range_start - 1] + else: + del self[i] + + if end is not None: + range_start = end + 1 + if range_start < range_end: + self.exclude(range_start, range_end) + break + + +class PersistentSequence(Sequence): + + def __init__(self, path, empty_value=None): + Sequence.__init__(self, empty_value=empty_value) + self._path = path + + if exists(self._path): + with file(self._path) as f: + self[:] = json.load(f) + + @property + def mtime(self): + if exists(self._path): + return os.stat(self._path).st_mtime + + def commit(self): + dir_path = dirname(self._path) + if dir_path and not exists(dir_path): + os.makedirs(dir_path) + with new_file(self._path) as f: + json.dump(self, f) + f.flush() + os.fsync(f.fileno()) + + +class Pool(object): + """Stack that keeps its iterators correct after changing content.""" + + QUEUED = 0 + ACTIVE = 1 + PASSED = 2 + + def __init__(self): + self._queue = collections.deque() + + def add(self, value): + self.remove(value) + self._queue.appendleft([Pool.QUEUED, value]) + + def remove(self, value): + for i, (state, existing) in enumerate(self._queue): + if existing == value: + del self._queue[i] + return state + + def get_state(self, value): + for state, existing in self._queue: + if existing == value: + return state + + def rewind(self): + for i in self._queue: + i[0] = Pool.QUEUED + + def __len__(self): + return len(self._queue) + + def __iter__(self): + for i in self._queue: + state, value = i + if state == Pool.PASSED: + continue + try: + i[0] = Pool.ACTIVE + yield value + finally: + i[0] = Pool.PASSED + + def __repr__(self): + return str([i[1] for i in self._queue]) + + +class _NullHandler(logging.Handler): + + def emit(self, record): + pass + + +class _NewFile(object): + + dst_path = None + + def __init__(self, **kwargs): + self._file = tempfile.NamedTemporaryFile(delete=False, **kwargs) + + @property + def name(self): + return self._file.name + + def close(self): + self._file.close() + if exists(self.name): + os.rename(self.name, self.dst_path) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.close() + + def __getattr__(self, name): + return getattr(self._file.file, name) + + +def _nb_read(stream): + import fcntl + + if stream is None: + return '' + fd = stream.fileno() + orig_flags = fcntl.fcntl(fd, fcntl.F_GETFL) + try: + fcntl.fcntl(fd, fcntl.F_SETFL, orig_flags | os.O_NONBLOCK) + return stream.read() + except Exception: + return '' + finally: + fcntl.fcntl(fd, fcntl.F_SETFL, orig_flags) + + +def _disable_logger(loggers): + for log_name in loggers: + logger = logging.getLogger(log_name) + logger.propagate = False + logger.addHandler(_NullHandler()) + + _default_lang = None diff --git a/sugar_network/toolkit/http.py b/sugar_network/toolkit/http.py index 30eb59e..e02ceda 100644 --- a/sugar_network/toolkit/http.py +++ b/sugar_network/toolkit/http.py @@ -26,8 +26,7 @@ from requests import Session from requests.exceptions import SSLError, ConnectionError, HTTPError from sugar_network import client, toolkit -from sugar_network.toolkit import coroutine, util -from sugar_network.toolkit import BUFFER_SIZE, exception, enforce +from sugar_network.toolkit import coroutine, enforce _logger = logging.getLogger('http') @@ -37,7 +36,6 @@ class Status(Exception): status = None headers = None - result = None class StatusPass(Status): @@ -108,8 +106,8 @@ class Client(object): session.verify = False if creds: uid, keyfile, self._get_profile = creds - session.headers['sugar_user'] = uid - session.headers['sugar_user_signature'] = _sign(keyfile, uid) + session.headers['X-SN-login'] = uid + session.headers['X-SN-signature'] = _sign(keyfile, uid) session.headers['accept-language'] = toolkit.default_lang() def __repr__(self): @@ -161,9 +159,9 @@ class Client(object): content_length = response.headers.get('Content-Length') if content_length: - chunk_size = min(int(content_length), BUFFER_SIZE) + chunk_size = min(int(content_length), toolkit.BUFFER_SIZE) else: - chunk_size = BUFFER_SIZE + chunk_size = toolkit.BUFFER_SIZE if dst is None: return response.iter_content(chunk_size=chunk_size) @@ -182,6 +180,9 @@ class Client(object): path = [''] if not isinstance(path, basestring): path = '/'.join([i.strip('/') for i in [self.api_url] + path]) + if isinstance(params, basestring): + path += '?' + params + params = None a_try = 0 while True: @@ -214,7 +215,8 @@ class Client(object): # If so, try to resend request. if a_try <= self._max_retries and method == 'GET': continue - error = content or 'No error message provided' + error = content or response.headers.get('x-sn-error') or \ + 'No error message provided' _logger.trace('Request failed, method=%s path=%r params=%r ' 'headers=%r status_code=%s error=%s', method, path, params, headers, response.status_code, @@ -225,23 +227,6 @@ class Client(object): return response def call(self, request, response=None): - params = request.copy() - method = params.pop('method') - document = params.pop('document') if 'document' in params else None - guid = params.pop('guid') if 'guid' in params else None - prop = params.pop('prop') if 'prop' in params else None - - if request.path is not None: - path = request.path - else: - path = [] - if document: - path.append(document) - if guid: - path.append(guid) - if prop: - path.append(prop) - if request.content_type == 'application/json': request.content = json.dumps(request.content) @@ -259,22 +244,20 @@ class Client(object): else: request.content = request.content_stream.read() headers['content-length'] = str(len(request.content)) - if request.accept_language: - headers['accept-language'] = request.accept_language[0] - if hasattr(request, 'environ'): - for env_key, key in ( - ('HTTP_IF_MODIFIED_SINCE', 'if-modified-since'), - ): + for env_key, key, value in ( + ('HTTP_IF_MODIFIED_SINCE', 'if-modified-since', None), + ('HTTP_ACCEPT_LANGUAGE', 'accept-language', + client.accept_language.value), + ('HTTP_ACCEPT_ENCODING', 'accept-encoding', None), + ): + if value is None: value = request.environ.get(env_key) - if value: - headers[key] = value - - reply = self.request(method, path, data=request.content, - params=params, headers=headers, allowed=[303], - allow_redirects=request.allow_redirects) + if value is not None: + headers[key] = value - if reply.status_code == 303: - raise Redirect(reply.headers.get('location')) + reply = self.request(request.method, request.path, + data=request.content, params=request.query or request, + headers=headers, allow_redirects=True) if response is not None: if 'transfer-encoding' in reply.headers: @@ -282,7 +265,7 @@ class Client(object): del reply.headers['transfer-encoding'] response.update(reply.headers) - if method != 'HEAD': + if request.method != 'HEAD': if reply.headers.get('Content-Type') == 'application/json': return json.loads(reply.content) else: @@ -319,13 +302,13 @@ class _Subscription(object): for a_try in (1, 0): stream = self._handshake() try: - line = util.readline(stream) + line = toolkit.readline(stream) enforce(line, 'Subscription aborted') break except Exception: if a_try == 0: raise - exception('Failed to read from %r subscription, ' + toolkit.exception('Failed to read from %r subscription, ' 'will resubscribe', self._client.api_url) self._content = None @@ -333,7 +316,8 @@ class _Subscription(object): try: return json.loads(line.split(' ', 1)[1]) except Exception: - exception('Failed to parse %r event from %r subscription', + toolkit.exception( + 'Failed to parse %r event from %r subscription', line, self._client.api_url) def _handshake(self, **params): diff --git a/sugar_network/toolkit/inotify.py b/sugar_network/toolkit/inotify.py index 26ad619..5475565 100644 --- a/sugar_network/toolkit/inotify.py +++ b/sugar_network/toolkit/inotify.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Aleksey Lim +# Copyright (C) 2012-2013 Aleksey Lim # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/sugar_network/toolkit/pipe.py b/sugar_network/toolkit/pipe.py index 17d887e..aec97d0 100644 --- a/sugar_network/toolkit/pipe.py +++ b/sugar_network/toolkit/pipe.py @@ -1,4 +1,4 @@ -# Copyright (C) 2010-2012 Aleksey Lim +# Copyright (C) 2010-2013 Aleksey Lim # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,7 +22,8 @@ import logging import threading from os.path import exists, dirname, basename -from sugar_network.toolkit import coroutine, util +from sugar_network import toolkit +from sugar_network.toolkit import coroutine _logger = logging.getLogger('pipe') @@ -176,7 +177,7 @@ def _setup_logging(path): log_dir = dirname(path) if not exists(log_dir): os.makedirs(log_dir) - path = util.unique_filename(log_dir, basename(path) + '.log') + path = toolkit.unique_filename(log_dir, basename(path) + '.log') logfile = file(path, 'a+') os.dup2(logfile.fileno(), sys.stdout.fileno()) diff --git a/sugar_network/toolkit/router.py b/sugar_network/toolkit/router.py new file mode 100644 index 0000000..9ecf556 --- /dev/null +++ b/sugar_network/toolkit/router.py @@ -0,0 +1,683 @@ +# Copyright (C) 2012-2013 Aleksey Lim +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import cgi +import json +import time +import types +import logging +import mimetypes +from bisect import bisect_left +from urllib import urlencode +from urlparse import parse_qsl, urlsplit +from email.utils import parsedate, formatdate +from os.path import isfile, split, splitext + +from sugar_network import toolkit +from sugar_network.toolkit import http, coroutine, enforce + + +_logger = logging.getLogger('router') + + +def route(method, path=None, cmd=None, **kwargs): + if path is None: + path = [] + enforce(method, 'Method should not be empty') + + def decorate(func): + func.route = (False, method, path, cmd, kwargs) + return func + + return decorate + + +def fallbackroute(method=None, path=None, **kwargs): + if path is None: + path = [] + enforce(not [i for i in path if i is None], + 'Wildcards is not allowed for fallbackroute') + + def decorate(func): + func.route = (True, method, path, None, kwargs) + return func + + return decorate + + +def preroute(func): + func.is_preroute = True + return func + + +def postroute(func): + func.is_postroute = True + return func + + +class ACL(object): + + INSYSTEM = 1 << 0 + ORIGINAL = 1 << 1 + + CREATE = 1 << 2 + WRITE = 1 << 3 + READ = 1 << 4 + DELETE = 1 << 5 + PUBLIC = CREATE | WRITE | READ | DELETE + + AUTH = 1 << 6 + AUTHOR = 1 << 7 + SUPERUSER = 1 << 8 + + LOCAL = 1 << 9 + CALC = 1 << 10 + + NAMES = { + CREATE: 'Create', + WRITE: 'Write', + READ: 'Read', + DELETE: 'Delete', + } + + +class Request(dict): + + environ = None + url = None + method = None + path = None + cmd = None + content = None + content_type = None + content_length = 0 + principal = None + + def __init__(self, environ=None, method=None, path=None, cmd=None, + **kwargs): + dict.__init__(self) + self._pos = 0 + self._dirty_query = False + + if environ is None: + self.environ = {} + self.method = method + self.path = path + self.cmd = cmd + self.update(kwargs) + return + + self.environ = environ + self.url = '/' + environ['PATH_INFO'].strip('/') + self.path = [i for i in self.url[1:].split('/') if i] + self.method = environ['REQUEST_METHOD'] + + enforce('..' not in self.path, 'Relative url path') + + query = environ.get('QUERY_STRING') or '' + for key, value in parse_qsl(query, keep_blank_values=True): + key = str(key) + param = self.get(key) + if type(param) is list: + param.append(value) + else: + if param is not None: + value = [param, value] + if key == 'cmd': + self.cmd = value + else: + dict.__setitem__(self, key, value) + if query: + self.url += '?' + query + + content_length = self.environ.get('CONTENT_LENGTH') + if content_length is not None: + self.content_length = int(content_length) + + content_type, __ = cgi.parse_header(environ.get('CONTENT_TYPE', '')) + self.content_type = content_type.lower() + if self.content_type == 'application/json': + self.content = json.load(environ['wsgi.input']) + + def __setitem__(self, key, value): + self._dirty_query = True + if key == 'cmd': + self.cmd = value + else: + dict.__setitem__(self, key, value) + + def __getitem__(self, key): + enforce(key in self, 'Cannot find %r request argument', key) + return self.get(key) + + @property + def resource(self): + if self.path: + return self.path[0] + + @property + def guid(self): + if len(self.path) > 1: + return self.path[1] + + @property + def prop(self): + if len(self.path) > 2: + return self.path[2] + + @property + def content_stream(self): + return self.environ.get('wsgi.input') + + @property + def static_prefix(self): + http_host = self.environ.get('HTTP_HOST') + if http_host: + return 'http://' + http_host + + @property + def if_modified_since(self): + value = parsedate(self.environ.get('HTTP_IF_MODIFIED_SINCE')) + if value is not None: + return time.mktime(value) + + @property + def accept_language(self): + return _parse_accept_language(self.environ.get('HTTP_ACCEPT_LANGUAGE')) + + @property + def accept_encoding(self): + return self.environ.get('HTTP_ACCEPT_ENCODING') + + @accept_encoding.setter + def accept_encoding(self, value): + self.environ['HTTP_ACCEPT_ENCODING'] = value + + @property + def query(self): + if self._dirty_query: + if self.cmd: + query = self.copy() + query['cmd'] = self.cmd + else: + query = self + self.environ['QUERY_STRING'] = urlencode(query, doseq=True) + self._dirty_query = False + return self.environ.get('QUERY_STRING') + + def read(self, size=None): + if self.content_stream is None: + return '' + rest = max(0, self.content_length - self._pos) + size = rest if size is None else min(rest, size) + result = self.content_stream.read(size) + if not result: + return '' + self._pos += len(result) + return result + + def __repr__(self): + return '<Request method=%s path=%r cmd=%s query=%r>' % \ + (self.method, self.path, self.cmd, dict(self)) + + +class Response(dict): + + status = '200 OK' + + def __init__(self, **kwargs): + dict.__init__(self, kwargs) + self.meta = {} + + @property + def content_length(self): + return int(self.get('content-length') or '0') + + @content_length.setter + def content_length(self, value): + self.set('content-length', value) + + @property + def content_type(self): + return self.get('content-type') + + @content_type.setter + def content_type(self, value): + if value: + self.set('content-type', value) + elif 'content-type' in self: + self.remove('content-type') + + @property + def last_modified(self): + return self.get('last-modified') + + @last_modified.setter + def last_modified(self, value): + self.set('last-modified', + formatdate(value, localtime=False, usegmt=True)) + + def items(self): + result = [] + for key, value in dict.items(self): + if type(value) in (list, tuple): + for i in value: + result.append((key, str(i))) + else: + result.append((key, str(value))) + return result + + def __repr__(self): + items = ['%s=%r' % i for i in self.items()] + return '<Response %s>' % ' '.join(items) + + def __contains__(self, key): + dict.__contains__(self, key.lower()) + + def __getitem__(self, key): + return self.get(key.lower()) + + def __setitem__(self, key, value): + return self.set(key.lower(), value) + + def __delitem__(self, key, value): + self.remove(key.lower()) + + def set(self, key, value): + dict.__setitem__(self, key, value) + + def remove(self, key): + dict.__delitem__(self, key) + + +class Blob(dict): + pass + + +class Router(object): + + def __init__(self, routes_model): + self._valid_origins = set() + self._invalid_origins = set() + self._host = None + self._routes = _Routes() + self._routes_model = routes_model + self._preroutes = [] + self._postroutes = [] + + processed = set() + cls = type(routes_model) + while cls is not None: + for name in dir(cls): + attr = getattr(cls, name) + if name in processed: + continue + if hasattr(attr, 'is_preroute'): + self._preroutes.append(getattr(routes_model, name)) + continue + elif hasattr(attr, 'is_postroute'): + self._postroutes.append(getattr(routes_model, name)) + continue + elif not hasattr(attr, 'route'): + continue + fallback, method, path, cmd, kwargs = attr.route + routes = self._routes + for i, part in enumerate(path): + enforce(i == 0 or not routes.fallback_ops or \ + (fallback and i == len(path) - 1), + 'Fallback route should not have sub-routes') + if part is None: + enforce(not fallback, 'Fallback route with wildcards') + if routes.wildcards is None: + routes.wildcards = _Routes(routes.parent) + routes = routes.wildcards + else: + routes = routes.setdefault(part, _Routes(routes)) + ops = routes.fallback_ops if fallback else routes.ops + route_ = _Route(getattr(routes_model, name), method, path, cmd, + **kwargs) + enforce(route_.op not in ops, 'Route %s already exists', + route_) + ops[route_.op] = route_ + processed.add(name) + cls = cls.__base__ + + def call(self, request, response): + result = None + try: + result = self._call(request, response) + + if isinstance(result, Blob): + if 'url' in result: + raise http.Redirect(result['url']) + + path = result['blob'] + enforce(isfile(path), 'No such file') + + mtime = result.get('mtime') or os.stat(path).st_mtime + if request.if_modified_since and mtime and \ + mtime <= request.if_modified_since: + raise http.NotModified() + response.last_modified = mtime + + response.content_type = result.get('mime_type') or \ + 'application/octet-stream' + + filename = result.get('filename') + if not filename: + filename = _filename(result.get('name') or + splitext(split(path)[-1])[0], + response.content_type) + response['Content-Disposition'] = \ + 'attachment; filename="%s"' % filename + + result = file(path, 'rb') + + if hasattr(result, 'read'): + if hasattr(result, 'fileno'): + response.content_length = os.fstat(result.fileno()).st_size + elif hasattr(result, 'seek'): + result.seek(0, 2) + response.content_length = result.tell() + result.seek(0) + result = _stream_reader(result) + finally: + _logger.trace('%s call: request=%s response=%r result=%r', + self, request, response, result) + return result + + def __repr__(self): + return '<Router %s>' % type(self._routes_model).__name__ + + def __call__(self, environ, start_response): + request = Request(environ) + response = Response() + + js_callback = None + if 'callback' in request: + js_callback = request.pop('callback') + + result = None + try: + if 'HTTP_ORIGIN' in request.environ: + enforce(self._assert_origin(request.environ), http.Forbidden, + 'Cross-site is not allowed for %r origin', + request.environ['HTTP_ORIGIN']) + response['Access-Control-Allow-Origin'] = \ + request.environ['HTTP_ORIGIN'] + result = self.call(request, response) + except http.StatusPass, error: + response.status = error.status + if error.headers: + response.update(error.headers) + response.content_type = None + except Exception, error: + toolkit.exception('Error while processing %r request', request.url) + if isinstance(error, http.Status): + response.status = error.status + response.update(error.headers or {}) + else: + response.status = '500 Internal Server Error' + if request.method == 'HEAD': + response.meta['error'] = str(error) + else: + result = {'error': str(error), + 'request': request.url, + } + response.content_type = 'application/json' + + result_streamed = isinstance(result, types.GeneratorType) + + if request.method == 'HEAD': + result_streamed = False + result = None + elif js_callback: + if result_streamed: + result = ''.join(result) + result_streamed = False + result = '%s(%s);' % (js_callback, json.dumps(result)) + response.content_length = len(result) + elif not result_streamed: + if response.content_type == 'application/json': + result = json.dumps(result) + if 'content-length' not in response: + response.content_length = len(result) if result else 0 + + for key, value in response.meta.items(): + response.set('X-SN-%s' % str(key), json.dumps(value)) + + start_response(response.status, response.items()) + + if request.method == 'HEAD': + enforce(result is None, 'HEAD responses should not contain body') + elif result_streamed: + for i in result: + yield i + elif result is not None: + yield result + + def _call(self, request, response): + route_ = self._resolve(request) + request.routes = self._routes_model + + for arg, cast in route_.arguments.items(): + value = request.get(arg) + if value is None: + if not hasattr(cast, '__call__'): + request[arg] = cast + continue + if not hasattr(cast, '__call__'): + cast = type(cast) + try: + request[arg] = _typecast(cast, value) + except Exception, error: + raise http.BadRequest( + 'Cannot typecast %r argument: %s' % (arg, error)) + kwargs = {} + for arg in route_.kwarg_names: + if arg == 'request': + kwargs[arg] = request + elif arg == 'response': + kwargs[arg] = response + elif arg not in kwargs: + kwargs[arg] = request.get(arg) + + for i in self._preroutes: + i(route_, request) + result = None + exception = None + try: + result = route_.callback(**kwargs) + except Exception, exception: + raise + else: + if not response.content_type: + if isinstance(result, Blob): + response.content_type = result.get('mime_type') + if not response.content_type: + response.content_type = route_.mime_type + finally: + for i in self._postroutes: + i(request, response, result, exception) + + return result + + def _resolve(self, request): + found_path = [False] + + def resolve_path(routes, path): + if not path: + if routes.ops: + found_path[0] = True + return routes.ops.get((request.method, request.cmd)) or \ + routes.fallback_ops.get((request.method, None)) or \ + routes.fallback_ops.get((None, None)) + subroutes = routes.get(path[0]) + if subroutes is None: + route_ = routes.fallback_ops.get((request.method, None)) or \ + routes.fallback_ops.get((None, None)) + if route_ is not None: + return route_ + for subroutes in (subroutes, routes.wildcards): + if subroutes is None: + continue + route_ = resolve_path(subroutes, path[1:]) + if route_ is not None: + return route_ + + route_ = resolve_path(self._routes, request.path) or \ + self._routes.fallback_ops.get((request.method, None)) or \ + self._routes.fallback_ops.get((None, None)) + if route_ is None: + if found_path[0]: + raise http.BadRequest('No such operation') + else: + raise http.NotFound('Path not found') + return route_ + + def _assert_origin(self, environ): + origin = environ['HTTP_ORIGIN'] + if origin in self._valid_origins: + return True + if origin in self._invalid_origins: + return False + + valid = True + if origin == 'null' or origin.startswith('file://'): + # True all time for local apps + pass + else: + if self._host is None: + http_host = environ['HTTP_HOST'].split(':', 1)[0] + self._host = coroutine.gethostbyname(http_host) + ip = coroutine.gethostbyname(urlsplit(origin).hostname) + valid = (self._host == ip) + + if valid: + _logger.info('%s allow cross-site for %r origin', self, origin) + self._valid_origins.add(origin) + else: + _logger.info('%s disallow cross-site for %r origin', self, origin) + self._invalid_origins.add(origin) + return valid + + +def _filename(names, mime_type): + if type(names) not in (list, tuple): + names = [names] + parts = [] + for name in names: + if isinstance(name, dict): + name = toolkit.gettext(name) + parts.append(''.join([i.capitalize() for i in str(name).split()])) + result = '-'.join(parts) + if mime_type: + if not mimetypes.inited: + mimetypes.init() + result += mimetypes.guess_extension(mime_type) or '' + return result.replace(os.sep, '') + + +def _stream_reader(stream): + try: + while True: + chunk = stream.read(toolkit.BUFFER_SIZE) + if not chunk: + break + yield chunk + finally: + if hasattr(stream, 'close'): + stream.close() + + +def _typecast(cast, value): + if cast is list or cast is tuple: + if isinstance(value, basestring): + if value: + return value.split(',') + else: + return () + return list(value) + if isinstance(value, (list, tuple)): + value = value[-1] + if cast is int: + if isinstance(value, basestring) and not value: + return 0 + return int(value) + if cast is bool: + if isinstance(value, basestring): + return value.strip().lower() in ('true', '1', 'on', '') + return bool(value) + return cast(value) + + +def _parse_accept_language(value): + if not value: + return [toolkit.default_lang()] + langs = [] + qualities = [] + for chunk in value.split(','): + lang, params = (chunk.split(';', 1) + [None])[:2] + lang = lang.strip() + if not lang: + continue + quality = 1 + if params: + params = params.split('=', 1) + if len(params) > 1 and params[0].strip() == 'q': + quality = float(params[1]) + index = bisect_left(qualities, quality) + qualities.insert(index, quality) + langs.insert(len(langs) - index, lang.lower().replace('_', '-')) + return langs + + +class _Routes(dict): + + def __init__(self, parent=None): + dict.__init__(self) + self.parent = parent + self.wildcards = None + self.ops = {} + self.fallback_ops = {} + + +class _Route(object): + + def __init__(self, callback, method, path, cmd, mime_type=None, acl=0, + arguments=None): + self.op = (method, cmd) + self.callback = callback + self.method = method + self.path = path + self.cmd = cmd + self.mime_type = mime_type + self.acl = acl + self.arguments = arguments or {} + self.kwarg_names = [] + + if hasattr(callback, 'im_func'): + callback = callback.im_func + if hasattr(callback, 'func_code'): + code = callback.func_code + # `1:` is for skipping the first, `self` or `cls`, argument + self.kwarg_names = code.co_varnames[1:code.co_argcount] + + def __repr__(self): + path = '/'.join(['*' if i is None else i for i in self.path]) + if self.cmd: + path += ('?cmd=%s' % self.cmd) + return '%s /%s (%s)' % (self.method, path, self.callback.__name__) diff --git a/sugar_network/toolkit/spec.py b/sugar_network/toolkit/spec.py index b914c5c..d51abe9 100644 --- a/sugar_network/toolkit/spec.py +++ b/sugar_network/toolkit/spec.py @@ -202,7 +202,7 @@ def ensure_requires(to_consider, to_apply): def intersect(x, y): l = max([parse_version(i) for i, __ in (x + y)]) r = min([[[sys.maxint]] if i is None else parse_version(i) \ - for __, i in [x + y]]) + for __, i in (x + y)]) return l is None or r is None or l < r for name, cond in to_apply.items(): diff --git a/sugar_network/toolkit/util.py b/sugar_network/toolkit/util.py deleted file mode 100644 index 107235b..0000000 --- a/sugar_network/toolkit/util.py +++ /dev/null @@ -1,678 +0,0 @@ -# Copyright (C) 2011-2012 Aleksey Lim -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -"""Swiss knife module.""" - -import os -import json -import errno -import logging -import hashlib -import tempfile -import collections -from os.path import exists, join, islink, isdir, dirname, basename, abspath -from os.path import lexists, isfile - -from sugar_network.toolkit import BUFFER_SIZE, cachedir, enforce - - -_logger = logging.getLogger('toolkit.util') - - -def init_logging(debug_level): - # pylint: disable-msg=W0212 - - logging.addLevelName(9, 'TRACE') - logging.addLevelName(8, 'HEARTBEAT') - - logging.Logger.trace = lambda self, message, *args, **kwargs: None - logging.Logger.heartbeat = lambda self, message, *args, **kwargs: None - - if debug_level < 3: - _disable_logger([ - 'requests.packages.urllib3.connectionpool', - 'requests.packages.urllib3.poolmanager', - 'requests.packages.urllib3.response', - 'requests.packages.urllib3', - 'inotify', - 'netlink', - 'sugar_stats', - '0install', - ]) - elif debug_level < 4: - logging.Logger.trace = lambda self, message, *args, **kwargs: \ - self._log(9, message, args, **kwargs) - _disable_logger(['sugar_stats']) - else: - logging.Logger.trace = lambda self, message, *args, **kwargs: \ - self._log(9, message, args, **kwargs) - logging.Logger.heartbeat = lambda self, message, *args, **kwargs: \ - self._log(8, message, args, **kwargs) - - -def ensure_key(path): - if not exists(path): - if not exists(dirname(path)): - os.makedirs(dirname(path)) - _logger.info('Create DSA key') - assert_call([ - '/usr/bin/ssh-keygen', '-q', '-t', 'dsa', '-f', path, - '-C', '', '-N', '']) - key = pubkey(path).split()[1] - return str(hashlib.sha1(key).hexdigest()) - - -def pubkey(path): - with file(path + '.pub') as f: - for line in f.readlines(): - line = line.strip() - if line.startswith('ssh-'): - return line - raise RuntimeError('No valid DSA public keys in %r' % path) - - -def iter_file(*path): - with file(join(*path), 'rb') as f: - while True: - chunk = f.read(BUFFER_SIZE) - if not chunk: - return - yield chunk - - -def readline(stream, limit=None): - line = bytearray() - while limit is None or len(line) < limit: - char = stream.read(1) - if not char: - break - line.append(char) - if char == '\n': - break - return bytes(line) - - -def default_route_exists(): - with file('/proc/self/net/route') as f: - # Skip header - f.readline() - while True: - line = f.readline() - if not line: - break - if int(line.split('\t', 2)[1], 16) == 0: - return True - - -def spawn(cmd_filename, *args): - _logger.trace('Spawn %s%r', cmd_filename, args) - - if os.fork(): - return - - os.execvp(cmd_filename, (cmd_filename,) + args) - - -def symlink(src, dst): - if not isfile(src): - _logger.debug('Cannot link %r to %r, source file is absent', src, dst) - return - - _logger.trace('Link %r to %r', src, dst) - - if lexists(dst): - os.unlink(dst) - elif not exists(dirname(dst)): - os.makedirs(dirname(dst)) - os.symlink(src, dst) - - -def svg_to_png(src_path, dst_path, width, height): - import rsvg - import cairo - - svg = rsvg.Handle(src_path) - - surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, width, height) - context = cairo.Context(surface) - scale = min( - float(width) / svg.props.width, - float(height) / svg.props.height) - context.scale(scale, scale) - svg.render_cairo(context) - - surface.write_to_png(dst_path) - - -def assert_call(cmd, stdin=None, **kwargs): - """Variant of `call` method with raising exception of errors. - - :param cmd: - commad to execute, might be string or argv list - :param stdin: - text that will be used as an input for executed process - - """ - return call(cmd, stdin=stdin, asserts=True, **kwargs) - - -def call(cmd, stdin=None, asserts=False, raw=False, error_cb=None, **kwargs): - """Convenient wrapper around subprocess call. - - Note, this function is intended for processes that output finite - and not big amount of text. - - :param cmd: - commad to execute, might be string or argv list - :param stdin: - text that will be used as an input for executed process - :param asserts: - whether to raise `RuntimeError` on fail execution status - :param error_cb: - call callback(stderr) on getting error exit status from the process - :returns: - `None` on errors, otherwise `str` value of stdout - - """ - import subprocess - - stdout, stderr = None, None - returncode = 1 - try: - logging.debug('Exec %r', cmd) - process = subprocess.Popen(cmd, stderr=subprocess.PIPE, - stdout=subprocess.PIPE, stdin=subprocess.PIPE, **kwargs) - if stdin is not None: - process.stdin.write(stdin) - process.stdin.close() - # Avoid using Popen.communicate() - # http://bugs.python.org/issue4216#msg77582 - process.wait() - stdout = _nb_read(process.stdout) - stderr = _nb_read(process.stderr) - if not raw: - stdout = stdout.strip() - stderr = stderr.strip() - returncode = process.returncode - enforce(returncode == 0, 'Exit status is an error') - logging.debug('Successfully executed stdout=%r stderr=%r', - stdout.split('\n'), stderr.split('\n')) - return stdout - except Exception, error: - logging.debug('Failed to execute error="%s" stdout=%r stderr=%r', - error, str(stdout).split('\n'), str(stderr).split('\n')) - if asserts: - if type(cmd) not in (str, unicode): - cmd = ' '.join(cmd) - raise RuntimeError('Failed to execute "%s" command: %s' % - (cmd, error)) - elif error_cb is not None: - error_cb(returncode, stdout, stderr) - - -def cptree(src, dst): - """Efficient version of copying directories. - - Function will try to make hard links for copying files at first and - will fallback to regular copying overwise. - - :param src: - path to the source directory - :param dst: - path to the new directory - - """ - import shutil - - if abspath(src) == abspath(dst): - return - - do_copy = [] - src = abspath(src) - dst = abspath(dst) - - def link(src, dst): - if not exists(dirname(dst)): - os.makedirs(dirname(dst)) - - if islink(src): - link_to = os.readlink(src) - os.symlink(link_to, dst) - elif isdir(src): - cptree(src, dst) - elif do_copy: - # The first hard link was not set, do regular copying for the rest - shutil.copy(src, dst) - else: - if exists(dst) and os.stat(src).st_ino == os.stat(dst).st_ino: - return - if os.access(src, os.W_OK): - try: - os.link(src, dst) - except OSError: - do_copy.append(True) - shutil.copy(src, dst) - shutil.copystat(src, dst) - else: - # Avoid copystat from not current users - shutil.copy(src, dst) - - if isdir(src): - for root, __, files in os.walk(src): - dst_root = join(dst, root[len(src):].lstrip(os.sep)) - if not exists(dst_root): - os.makedirs(dst_root) - for i in files: - link(join(root, i), join(dst_root, i)) - else: - link(src, dst) - - -def new_file(path, mode=0644): - """Atomic new file creation. - - Method will create temporaty file in the same directory as the specified - one. When file object associated with this temporaty file will be closed, - temporaty file will be renamed to the final destination. - - :param path: - path to save final file to - :param mode: - mode for new file - :returns: - file object - - """ - result = _NewFile(dir=dirname(path), prefix=basename(path)) - result.dst_path = path - os.fchmod(result.fileno(), mode) - return result - - -def unique_filename(root, filename): - path = join(root, filename) - if exists(path): - name, suffix = os.path.splitext(filename) - for dup_num in xrange(1, 255): - path = join(root, name + '_' + str(dup_num) + suffix) - if not exists(path): - break - else: - raise RuntimeError('Cannot find unique filename for %r' % - join(root, filename)) - return path - - -def TemporaryFile(*args, **kwargs): - if cachedir.value: - if not exists(cachedir.value): - os.makedirs(cachedir.value) - kwargs['dir'] = cachedir.value - return tempfile.TemporaryFile(*args, **kwargs) - - -class NamedTemporaryFile(object): - - def __init__(self, *args, **kwargs): - if cachedir.value: - if not exists(cachedir.value): - os.makedirs(cachedir.value) - kwargs['dir'] = cachedir.value - self._file = tempfile.NamedTemporaryFile(*args, **kwargs) - - def close(self): - try: - self._file.close() - except OSError, error: - if error.errno != errno.ENOENT: - raise - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close() - - def __getattr__(self, name): - return getattr(self._file, name) - - -class Seqno(object): - """Sequence number counter with persistent storing in a file.""" - - def __init__(self, path): - """ - :param path: - path to file to [re]store seqno value - - """ - self._path = path - self._value = 0 - - if exists(path): - with file(path) as f: - self._value = int(f.read().strip()) - - self._orig_value = self._value - - @property - def value(self): - """Current seqno value.""" - return self._value - - def next(self): - """Incerement seqno. - - :returns: - new seqno value - - """ - self._value += 1 - return self._value - - def commit(self): - """Store current seqno value in a file. - - :returns: - `True` if commit was happened - - """ - if self._value == self._orig_value: - return False - with new_file(self._path) as f: - f.write(str(self._value)) - f.flush() - os.fsync(f.fileno()) - self._orig_value = self._value - return True - - -class Sequence(list): - """List of sorted and non-overlapping ranges. - - List items are ranges, [`start`, `stop']. If `start` or `stop` - is `None`, it means the beginning or ending of the entire scale. - - """ - - def __init__(self, value=None, empty_value=None): - """ - :param value: - default value to initialize range - :param empty_value: - if not `None`, the initial value for empty range - - """ - if empty_value is None: - self._empty_value = [] - else: - self._empty_value = [empty_value] - - if value: - self.extend(value) - else: - self.clear() - - def __contains__(self, value): - for start, end in self: - if value >= start and (end is None or value <= end): - return True - else: - return False - - @property - def empty(self): - """Is timeline in the initial state.""" - return self == self._empty_value - - def clear(self): - """Reset range to the initial value.""" - self[:] = self._empty_value - - def stretch(self): - """Remove all holes between the first and the last items.""" - if self: - self[:] = [[self[0][0], self[-1][-1]]] - - def include(self, start, end=None): - """Include specified range. - - :param start: - either including range start or a list of - (`start`, `end`) pairs - :param end: - including range end - - """ - if issubclass(type(start), collections.Iterable): - for range_start, range_end in start: - self._include(range_start, range_end) - elif start is not None: - self._include(start, end) - - def exclude(self, start, end=None): - """Exclude specified range. - - :param start: - either excluding range start or a list of - (`start`, `end`) pairs - :param end: - excluding range end - - """ - if issubclass(type(start), collections.Iterable): - for range_start, range_end in start: - self._exclude(range_start, range_end) - else: - enforce(end is not None) - self._exclude(start, end) - - def _include(self, range_start, range_end): - if range_start is None: - range_start = 1 - - range_start_new = None - range_start_i = 0 - - for range_start_i, (start, end) in enumerate(self): - if range_end is not None and start - 1 > range_end: - break - if (range_end is None or start - 1 <= range_end) and \ - (end is None or end + 1 >= range_start): - range_start_new = min(start, range_start) - break - else: - range_start_i += 1 - - if range_start_new is None: - self.insert(range_start_i, [range_start, range_end]) - return - - range_end_new = range_end - range_end_i = range_start_i - for i, (start, end) in enumerate(self[range_start_i:]): - if range_end is not None and start - 1 > range_end: - break - if range_end is None or end is None: - range_end_new = None - else: - range_end_new = max(end, range_end) - range_end_i = range_start_i + i - - del self[range_start_i:range_end_i] - self[range_start_i] = [range_start_new, range_end_new] - - def _exclude(self, range_start, range_end): - if range_start is None: - range_start = 1 - enforce(range_end is not None) - enforce(range_start <= range_end and range_start > 0, - 'Start value %r is less than 0 or not less than %r', - range_start, range_end) - - for i, interval in enumerate(self): - start, end = interval - - if end is not None and end < range_start: - # Current `interval` is below new one - continue - - if range_end is not None and range_end < start: - # Current `interval` is above new one - continue - - if end is None or end > range_end: - # Current `interval` will exist after changing - self[i] = [range_end + 1, end] - if start < range_start: - self.insert(i, [start, range_start - 1]) - else: - if start < range_start: - self[i] = [start, range_start - 1] - else: - del self[i] - - if end is not None: - range_start = end + 1 - if range_start < range_end: - self.exclude(range_start, range_end) - break - - -class PersistentSequence(Sequence): - - def __init__(self, path, empty_value=None): - Sequence.__init__(self, empty_value=empty_value) - self._path = path - - if exists(self._path): - with file(self._path) as f: - self[:] = json.load(f) - - @property - def mtime(self): - if exists(self._path): - return os.stat(self._path).st_mtime - - def commit(self): - dir_path = dirname(self._path) - if dir_path and not exists(dir_path): - os.makedirs(dir_path) - with new_file(self._path) as f: - json.dump(self, f) - f.flush() - os.fsync(f.fileno()) - - -class Pool(object): - """Stack that keeps its iterators correct after changing content.""" - - QUEUED = 0 - ACTIVE = 1 - PASSED = 2 - - def __init__(self): - self._queue = collections.deque() - - def add(self, value): - self.remove(value) - self._queue.appendleft([Pool.QUEUED, value]) - - def remove(self, value): - for i, (state, existing) in enumerate(self._queue): - if existing == value: - del self._queue[i] - return state - - def get_state(self, value): - for state, existing in self._queue: - if existing == value: - return state - - def rewind(self): - for i in self._queue: - i[0] = Pool.QUEUED - - def __len__(self): - return len(self._queue) - - def __iter__(self): - for i in self._queue: - state, value = i - if state == Pool.PASSED: - continue - try: - i[0] = Pool.ACTIVE - yield value - finally: - i[0] = Pool.PASSED - - def __repr__(self): - return str([i[1] for i in self._queue]) - - -class _NullHandler(logging.Handler): - - def emit(self, record): - pass - - -class _NewFile(object): - - dst_path = None - - def __init__(self, **kwargs): - self._file = tempfile.NamedTemporaryFile(delete=False, **kwargs) - - @property - def name(self): - return self._file.name - - def close(self): - self._file.close() - if exists(self.name): - os.rename(self.name, self.dst_path) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.close() - - def __getattr__(self, name): - return getattr(self._file.file, name) - - -def _nb_read(stream): - import fcntl - - if stream is None: - return '' - fd = stream.fileno() - orig_flags = fcntl.fcntl(fd, fcntl.F_GETFL) - try: - fcntl.fcntl(fd, fcntl.F_SETFL, orig_flags | os.O_NONBLOCK) - return stream.read() - except Exception: - return '' - finally: - fcntl.fcntl(fd, fcntl.F_SETFL, orig_flags) - - -def _disable_logger(loggers): - for log_name in loggers: - logger = logging.getLogger(log_name) - logger.propagate = False - logger.addHandler(_NullHandler()) diff --git a/tests/__init__.py b/tests/__init__.py index 84378c6..f4eed96 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -16,19 +16,17 @@ from os.path import dirname, join, exists, abspath, isfile from M2Crypto import DSA from gevent import monkey -from sugar_network.toolkit import coroutine, http, mountpoints, util, Option, pipe -from sugar_network.db.router import Router -from sugar_network.client import journal, IPCRouter, commands -from sugar_network.client.commands import ClientCommands +from sugar_network.toolkit import coroutine, http, mountpoints, Option, pipe +from sugar_network.toolkit.router import Router +from sugar_network.client import journal, routes as client_routes +from sugar_network.client.routes import ClientRoutes from sugar_network import db, client, node, toolkit -from sugar_network.db import env from sugar_network.client import injector, solver -from sugar_network.resources.user import User -from sugar_network.resources.context import Context -from sugar_network.resources.implementation import Implementation -from sugar_network.node.master import MasterCommands -from sugar_network.node import stats_user, stats_node, obs, auth, slave, downloads -from sugar_network.resources.volume import Volume +from sugar_network.model.user import User +from sugar_network.model.context import Context +from sugar_network.model.implementation import Implementation +from sugar_network.node.master import MasterRoutes +from sugar_network.node import stats_user, stats_node, obs, slave, downloads root = abspath(dirname(__file__)) @@ -84,7 +82,6 @@ class Test(unittest.TestCase): db.index_flush_threshold.value = 1 node.find_limit.value = 1024 node.data_root.value = tmpdir - node.static_url.value = None node.files_root.value = None node.sync_layers.value = None db.index_write_queue.value = 10 @@ -96,7 +93,7 @@ class Test(unittest.TestCase): client.layers.value = None client.cache_limit.value = 0 client.cache_lifetime.value = 0 - commands._RECONNECT_TIMEOUT = 0 + client_routes._RECONNECT_TIMEOUT = 0 mountpoints._connects.clear() mountpoints._found.clear() mountpoints._COMPLETE_MOUNT_TIMEOUT = .1 @@ -109,7 +106,6 @@ class Test(unittest.TestCase): obs._client = None obs._repos = {'base': [], 'presolve': []} http._RECONNECTION_NUMBER = 0 - auth.reset() toolkit.cachedir.value = tmpdir + '/tmp' injector.invalidate_solutions(None) injector._pms_path = None @@ -119,12 +115,12 @@ class Test(unittest.TestCase): pipe._pipe = None pipe._trace = None - Volume.RESOURCES = [ - 'sugar_network.resources.user', - 'sugar_network.resources.context', - 'sugar_network.resources.artifact', - 'sugar_network.resources.implementation', - 'sugar_network.resources.report', + db.Volume.model = [ + 'sugar_network.model.user', + 'sugar_network.model.context', + 'sugar_network.model.artifact', + 'sugar_network.model.implementation', + 'sugar_network.model.report', ] if tmp_root is None: @@ -146,8 +142,8 @@ class Test(unittest.TestCase): def tearDown(self): self.stop_nodes() - while Volume._flush_pool: - Volume._flush_pool.pop().close() + while db.Volume._flush_pool: + db.Volume._flush_pool.pop().close() while self._overriden: mod, name, old_handler = self._overriden.pop() setattr(mod, name, old_handler) @@ -224,7 +220,7 @@ class Test(unittest.TestCase): os.utime(join(root, i), (ts, ts)) def zips(self, *items): - with util.NamedTemporaryFile() as f: + with toolkit.NamedTemporaryFile() as f: bundle = zipfile.ZipFile(f.name, 'w') for i in items: if isinstance(i, basestring): @@ -269,20 +265,21 @@ class Test(unittest.TestCase): def start_master(self, classes=None): if classes is None: classes = [User, Context, Implementation] - self.node_volume = Volume('master', classes) - cp = MasterCommands('guid', self.node_volume) + self.node_volume = db.Volume('master', classes) + cp = MasterRoutes('guid', self.node_volume) + r = Router(cp) self.node = coroutine.WSGIServer(('127.0.0.1', 8888), Router(cp)) coroutine.spawn(self.node.serve_forever) coroutine.dispatch(.1) return self.node_volume - def fork_master(self, classes=None): + def fork_master(self, classes=None, routes=MasterRoutes): if classes is None: classes = [User, Context, Implementation] def node(): - volume = Volume('master', classes) - cp = MasterCommands('guid', volume) + volume = db.Volume('master', classes) + cp = routes('guid', volume) node = coroutine.WSGIServer(('127.0.0.1', 8888), Router(cp)) node.serve_forever() @@ -290,13 +287,13 @@ class Test(unittest.TestCase): coroutine.sleep(.1) return pid - def start_client(self, classes=None): + def start_client(self, classes=None, routes=ClientRoutes): if classes is None: classes = [User, Context, Implementation] - volume = Volume('client', classes) - commands = ClientCommands(volume, client.api_url.value) + volume = db.Volume('client', classes) + commands = routes(volume, client.api_url.value) self.client = coroutine.WSGIServer( - ('127.0.0.1', client.ipc_port.value), IPCRouter(commands)) + ('127.0.0.1', client.ipc_port.value), Router(commands)) coroutine.spawn(self.client.serve_forever) coroutine.dispatch() return volume @@ -305,11 +302,11 @@ class Test(unittest.TestCase): if classes is None: classes = [User, Context, Implementation] self.start_master(classes) - volume = Volume('client', classes) - commands = ClientCommands(volume, client.api_url.value) + volume = db.Volume('client', classes) + commands = ClientRoutes(volume, client.api_url.value) self.wait_for_events(commands, event='inline', state='online').wait() self.client = coroutine.WSGIServer( - ('127.0.0.1', client.ipc_port.value), IPCRouter(commands)) + ('127.0.0.1', client.ipc_port.value), Router(commands)) coroutine.spawn(self.client.serve_forever) coroutine.dispatch() return volume @@ -317,10 +314,10 @@ class Test(unittest.TestCase): def start_offline_client(self, classes=None): if classes is None: classes = [User, Context, Implementation] - volume = Volume('client', classes) - commands = ClientCommands(volume) + volume = db.Volume('client', classes) + commands = ClientRoutes(volume) self.client = coroutine.WSGIServer( - ('127.0.0.1', client.ipc_port.value), IPCRouter(commands)) + ('127.0.0.1', client.ipc_port.value), Router(commands)) coroutine.spawn(self.client.serve_forever) coroutine.dispatch() return volume @@ -341,8 +338,8 @@ class Test(unittest.TestCase): node.find_limit.value = 1024 db.index_write_queue.value = 10 - volume = Volume('remote', classes or [User, Context, Implementation]) - cp = MasterCommands('guid', volume) + volume = db.Volume('remote', classes or [User, Context, Implementation]) + cp = MasterRoutes('guid', volume) httpd = coroutine.WSGIServer(('127.0.0.1', 8888), Router(cp)) try: coroutine.joinall([ @@ -380,7 +377,7 @@ class Test(unittest.TestCase): logging.basicConfig(level=logging.DEBUG, filename=join(tmpdir, '%s.log' % fork_num), format='%(asctime)s %(levelname)s %(name)s: %(message)s') - util.init_logging(10) + toolkit.init_logging(10) sys.stdout.flush() sys.stderr.flush() diff --git a/tests/data/node/implementation/im/implementation/data b/tests/data/node/implementation/im/implementation/data index b933ad2..d8d70ef 100644 --- a/tests/data/node/implementation/im/implementation/data +++ b/tests/data/node/implementation/im/implementation/data @@ -1 +1 @@ -{"seqno": 5, "mime_type": "application/octet-stream", "digest": "fdb59f1ebd6ee26a00396747b3a733f4fd274604"}
\ No newline at end of file +{"seqno": 5, "mime_type": "application/octet-stream", "digest": "fdb59f1ebd6ee26a00396747b3a733f4fd274604", "spec": {"*-*": {"commands": {"activity": {"exec": "true"}}, "extract": "Chat.activity"}}}
\ No newline at end of file diff --git a/tests/data/node/implementation/im/implementation/spec b/tests/data/node/implementation/im/implementation/spec deleted file mode 100644 index c9ec14a..0000000 --- a/tests/data/node/implementation/im/implementation/spec +++ /dev/null @@ -1 +0,0 @@ -{"seqno": 3, "value": {"*-*": {"commands": {"activity": {"exec": "true"}}, "extract": "Chat.activity"}}}
\ No newline at end of file diff --git a/tests/data/node/implementation/im/implementation2/data b/tests/data/node/implementation/im/implementation2/data index b933ad2..c11bbb8 100644 --- a/tests/data/node/implementation/im/implementation2/data +++ b/tests/data/node/implementation/im/implementation2/data @@ -1 +1 @@ -{"seqno": 5, "mime_type": "application/octet-stream", "digest": "fdb59f1ebd6ee26a00396747b3a733f4fd274604"}
\ No newline at end of file +{"seqno": 5, "mime_type": "application/octet-stream", "digest": "fdb59f1ebd6ee26a00396747b3a733f4fd274604", "spec": {"*-*": {"commands": {"activity": {"exec": "true"}}, "extract": "Chat.activity", "requires": {"dep1": {}, "dep2": {}, "dep3": {}}}}}
\ No newline at end of file diff --git a/tests/data/node/implementation/im/implementation2/spec b/tests/data/node/implementation/im/implementation2/spec deleted file mode 100644 index 89f8d50..0000000 --- a/tests/data/node/implementation/im/implementation2/spec +++ /dev/null @@ -1 +0,0 @@ -{"seqno": 3, "value": {"*-*": {"commands": {"activity": {"exec": "true"}}, "extract": "Chat.activity", "requires": {"dep1": {}, "dep2": {}, "dep3": {}}}}}
\ No newline at end of file diff --git a/tests/integration/master_personal.py b/tests/integration/master_personal.py index 718659b..20b5376 100755 --- a/tests/integration/master_personal.py +++ b/tests/integration/master_personal.py @@ -15,7 +15,7 @@ from __init__ import tests, src_root from sugar_network.client import Client, sugar_uid from sugar_network.toolkit.rrd import Rrd -from sugar_network.toolkit import util, coroutine +from sugar_network.toolkit import coroutine # /tmp might be on tmpfs wich returns 0 bytes for free mem all time diff --git a/tests/integration/master_slave.py b/tests/integration/master_slave.py index 946034e..217997e 100755 --- a/tests/integration/master_slave.py +++ b/tests/integration/master_slave.py @@ -15,7 +15,7 @@ from __init__ import tests, src_root from sugar_network.client import Client from sugar_network.toolkit.rrd import Rrd -from sugar_network.toolkit import util, coroutine +from sugar_network.toolkit import coroutine # /tmp might be on tmpfs wich returns 0 bytes for free mem all time diff --git a/tests/integration/node_client.py b/tests/integration/node_client.py index 4d51883..f7981c8 100755 --- a/tests/integration/node_client.py +++ b/tests/integration/node_client.py @@ -10,8 +10,9 @@ from os.path import exists, join from __init__ import tests, src_root +from sugar_network import toolkit from sugar_network.client import IPCClient, Client -from sugar_network.toolkit import coroutine, util +from sugar_network.toolkit import coroutine class NodeClientTest(tests.Test): @@ -20,7 +21,7 @@ class NodeClientTest(tests.Test): tests.Test.setUp(self) os.makedirs('mnt') - util.cptree(src_root + '/tests/data/node', 'node') + toolkit.cptree(src_root + '/tests/data/node', 'node') self.client_pid = None self.node_pid = self.popen([join(src_root, 'sugar-network-node'), '-F', 'start', @@ -122,7 +123,7 @@ class NodeClientTest(tests.Test): if ipc.get(cmd='status')['route'] == 'offline': self.wait_for_events(ipc, event='inline', state='online').wait() - result = util.assert_call(cmd, stdin=json.dumps(stdin)) + result = toolkit.assert_call(cmd, stdin=json.dumps(stdin)) if result and '--porcelain' not in cmd: result = json.loads(result) return result diff --git a/tests/integration/node_packages.py b/tests/integration/node_packages.py index 0089e3d..e6ef11e 100755 --- a/tests/integration/node_packages.py +++ b/tests/integration/node_packages.py @@ -16,10 +16,10 @@ from __init__ import tests, src_root from sugar_network import db, client from sugar_network.client import Client, IPCClient -from sugar_network.db.router import Router, route from sugar_network.node.obs import obs_url +from sugar_network.toolkit.router import Router, route, fallbackroute from sugar_network.toolkit.rrd import Rrd -from sugar_network.toolkit import util, coroutine +from sugar_network.toolkit import coroutine # /tmp might be on tmpfs wich returns 0 bytes for free mem all time @@ -39,11 +39,10 @@ class NodePackagesSlaveTest(tests.Test): def test_packages(self): - class OBS(db.CommandsProcessor): + class OBS(object): - @route('GET', '/build') + @fallbackroute('GET', ['build'], mime_type='text/xml') def build(self, request, response): - response.content_type = 'text/xml' if request.path == ['build', 'base']: return '<directory><entry name="Fedora-14"/></directory>' elif request.path == ['build', 'base', 'Fedora-14']: @@ -53,12 +52,11 @@ class NodePackagesSlaveTest(tests.Test): elif request.path == ['build', 'presolve', 'OLPC-11.3.1']: return '<directory><entry name="i586"/></directory>' - @route('GET', '/resolve') + @fallbackroute('GET', ['resolve'], mime_type='text/xml') def resolve(self, request, response): - response.content_type = 'text/xml' return '<resolve><binary name="rpm" url="http://127.0.0.1:9999/packages/rpm" arch="arch"/></resolve>' - @route('GET', '/packages') + @fallbackroute('GET', ['packages'], mime_type='text/plain') def packages(self, request, response): return 'package_content' diff --git a/tests/units/__main__.py b/tests/units/__main__.py index 22664cb..1df3e3e 100644 --- a/tests/units/__main__.py +++ b/tests/units/__main__.py @@ -5,7 +5,7 @@ from __init__ import tests from toolkit.__main__ import * from db.__main__ import * from node.__main__ import * -from resources.__main__ import * +from model.__main__ import * from client.__main__ import * if __name__ == '__main__': diff --git a/tests/units/client/__main__.py b/tests/units/client/__main__.py index fd37288..fc1d045 100644 --- a/tests/units/client/__main__.py +++ b/tests/units/client/__main__.py @@ -3,12 +3,12 @@ from __init__ import tests from clones import * -from commands import * +from routes import * from injector import * from journal import * -from offline_commands import * -from online_commands import * -from server_commands import * +from offline_routes import * +from online_routes import * +from server_routes import * from solver import * from cache import * diff --git a/tests/units/client/clones.py b/tests/units/client/clones.py index b9178a1..974adca 100755 --- a/tests/units/client/clones.py +++ b/tests/units/client/clones.py @@ -8,18 +8,18 @@ from os.path import abspath, lexists, exists from __init__ import tests -from sugar_network.resources.user import User -from sugar_network.resources.context import Context +from sugar_network import db, model +from sugar_network.model.user import User +from sugar_network.model.context import Context from sugar_network.client import clones -from sugar_network.toolkit import coroutine, util -from sugar_network.resources.volume import Volume +from sugar_network.toolkit import coroutine class CloneTest(tests.Test): def setUp(self): tests.Test.setUp(self) - self.volume = Volume('local', [User, Context]) + self.volume = db.Volume('local', [User, Context]) self.job = None def tearDown(self): @@ -371,7 +371,7 @@ class CloneTest(tests.Test): assert not lexists('share/mime/application/x-foo-bar.xml') def test_Sync(self): - volume = Volume('client') + volume = db.Volume('client', model.RESOURCES) volume['context'].create({ 'guid': 'context1', 'type': 'activity', @@ -409,7 +409,7 @@ class CloneTest(tests.Test): self.assertEqual(0, volume['context'].get('context3')['clone']) def test_SyncByMtime(self): - volume = Volume('client') + volume = db.Volume('client', model.RESOURCES) volume['context'].create({ 'guid': 'context', 'type': 'activity', diff --git a/tests/units/client/injector.py b/tests/units/client/injector.py index 256cfa4..c198f81 100755 --- a/tests/units/client/injector.py +++ b/tests/units/client/injector.py @@ -17,9 +17,9 @@ from __init__ import tests from sugar_network.client import journal from sugar_network.toolkit import coroutine, enforce, pipe as pipe_, lsb_release from sugar_network.node import obs -from sugar_network.resources.user import User -from sugar_network.resources.context import Context -from sugar_network.resources.implementation import Implementation +from sugar_network.model.user import User +from sugar_network.model.context import Context +from sugar_network.model.implementation import Implementation from sugar_network.client import IPCClient, packagekit, injector, clones, solver from sugar_network import client diff --git a/tests/units/client/journal.py b/tests/units/client/journal.py index ffb6e4f..fd36a2a 100755 --- a/tests/units/client/journal.py +++ b/tests/units/client/journal.py @@ -12,6 +12,7 @@ from __init__ import tests from sugar_network import db from sugar_network.client import journal, ipc_port +from sugar_network.toolkit.router import Request, Response class JournalTest(tests.Test): @@ -76,7 +77,7 @@ class JournalTest(tests.Test): self.assertEqual('data', file(self.ds.get_filename(guid)).read()) def test_Update(self): - ds = journal.Commands() + ds = journal.Routes() self.touch(('preview', 'preview1')) ds.journal_update('guid', StringIO('data1'), title='title1', description='description1', preview={'blob': 'preview'}) @@ -97,76 +98,73 @@ class JournalTest(tests.Test): def test_FindRequest(self): url = 'http://127.0.0.1:%s/journal/' % ipc_port.value - ds = journal.Commands() + ds = journal.Routes() ds.journal_update('guid1', StringIO('data1'), title='title1', description='description1', preview=StringIO('preview1')) ds.journal_update('guid2', StringIO('data2'), title='title2', description='description2', preview=StringIO('preview2')) ds.journal_update('guid3', StringIO('data3'), title='title3', description='description3', preview=StringIO('preview3')) - request = db.Request() + request = Request(reply=['uid', 'title', 'description', 'preview']) request.path = ['journal'] - response = db.Response() + response = Response() self.assertEqual([ {'guid': 'guid1', 'title': 'title1', 'description': 'description1', 'preview': url + 'guid1/preview'}, {'guid': 'guid2', 'title': 'title2', 'description': 'description2', 'preview': url + 'guid2/preview'}, {'guid': 'guid3', 'title': 'title3', 'description': 'description3', 'preview': url + 'guid3/preview'}, ], - ds.journal(request, response)['result']) - self.assertEqual('application/json', response.content_type) + ds.journal_find(request, response)['result']) - request = db.Request(offset=1, limit=1) + request = Request(offset=1, limit=1, reply=['uid', 'title', 'description', 'preview']) request.path = ['journal'] self.assertEqual([ {'guid': 'guid2', 'title': 'title2', 'description': 'description2', 'preview': url + 'guid2/preview'}, ], - ds.journal(request, response)['result']) + ds.journal_find(request, response)['result']) - request = db.Request(query='title3') + request = Request(query='title3', reply=['uid', 'title', 'description', 'preview']) request.path = ['journal'] self.assertEqual([ {'guid': 'guid3', 'title': 'title3', 'description': 'description3', 'preview': url + 'guid3/preview'}, ], - ds.journal(request, response)['result']) + ds.journal_find(request, response)['result']) - request = db.Request(order_by='+title') + request = Request(order_by=['+title'], reply=['uid', 'title', 'description', 'preview']) request.path = ['journal'] self.assertEqual([ {'guid': 'guid3', 'title': 'title3', 'description': 'description3', 'preview': url + 'guid3/preview'}, {'guid': 'guid2', 'title': 'title2', 'description': 'description2', 'preview': url + 'guid2/preview'}, {'guid': 'guid1', 'title': 'title1', 'description': 'description1', 'preview': url + 'guid1/preview'}, ], - ds.journal(request, response)['result']) + ds.journal_find(request, response)['result']) def test_GetRequest(self): url = 'http://127.0.0.1:%s/journal/' % ipc_port.value - ds = journal.Commands() + ds = journal.Routes() ds.journal_update('guid1', StringIO('data1'), title='title1', description='description1', preview=StringIO('preview1')) - request = db.Request() + request = Request() request.path = ['journal', 'guid1'] - response = db.Response() + response = Response() self.assertEqual( {'guid': 'guid1', 'title': 'title1', 'description': 'description1', 'preview': url + 'guid1/preview'}, - ds.journal(request, response)) - self.assertEqual('application/json', response.content_type) + ds.journal_get(request, response)) def test_GetPropRequest(self): - ds = journal.Commands() + ds = journal.Routes() ds.journal_update('guid1', StringIO('data1'), title='title1', description='description1', preview=StringIO('preview1')) - request = db.Request() + request = Request() request.path = ['journal', 'guid1', 'title'] - response = db.Response() - self.assertEqual('title1', ds.journal(request, response)) - self.assertEqual('application/json', response.content_type) + response = Response() + self.assertEqual('title1', ds.journal_get_prop(request, response)) - request = db.Request() + request = Request() request.path = ['journal', 'guid1', 'preview'] - response = db.Response() + response = Response() self.assertEqual({ 'mime_type': 'image/png', 'blob': '.sugar/default/datastore/gu/guid1/metadata/preview', - }, ds.journal(request, response)) + }, ds.journal_get_preview(request, response)) self.assertEqual(None, response.content_type) diff --git a/tests/units/client/offline_commands.py b/tests/units/client/offline_routes.py index 69d5f2a..ced30d1 100755 --- a/tests/units/client/offline_commands.py +++ b/tests/units/client/offline_routes.py @@ -5,21 +5,21 @@ from os.path import exists from __init__ import tests, src_root -from sugar_network import client +from sugar_network import client, model from sugar_network.client import IPCClient, clones -from sugar_network.client.commands import ClientCommands -from sugar_network.client import IPCRouter -from sugar_network.resources.volume import Volume +from sugar_network.client.routes import ClientRoutes +from sugar_network.db import Volume +from sugar_network.toolkit.router import Router from sugar_network.toolkit import coroutine, http -class OfflineCommandsTest(tests.Test): +class OfflineRoutes(tests.Test): def setUp(self): tests.Test.setUp(self) - self.home_volume = Volume('db') - commands = ClientCommands(self.home_volume) - server = coroutine.WSGIServer(('127.0.0.1', client.ipc_port.value), IPCRouter(commands)) + self.home_volume = Volume('db', model.RESOURCES) + commands = ClientRoutes(self.home_volume) + server = coroutine.WSGIServer(('127.0.0.1', client.ipc_port.value), Router(commands)) coroutine.spawn(server.serve_forever) coroutine.dispatch() @@ -127,9 +127,9 @@ class OfflineCommandsTest(tests.Test): job.kill() self.assertEqual([ - {'guid': guid, 'document': 'context', 'event': 'create'}, - {'guid': guid, 'document': 'context', 'event': 'update'}, - {'guid': guid, 'event': 'delete', 'document': 'context'}, + {'guid': guid, 'resource': 'context', 'event': 'create'}, + {'guid': guid, 'resource': 'context', 'event': 'update'}, + {'guid': guid, 'event': 'delete', 'resource': 'context'}, ], events) diff --git a/tests/units/client/online_commands.py b/tests/units/client/online_routes.py index 692fa38..10d9b4b 100755 --- a/tests/units/client/online_commands.py +++ b/tests/units/client/online_routes.py @@ -11,24 +11,26 @@ from os.path import exists from __init__ import tests, src_root -from sugar_network import client, db -from sugar_network.client import IPCClient, journal, clones, injector, commands +from sugar_network import client, db, model +from sugar_network.client import IPCClient, journal, clones, injector, routes from sugar_network.toolkit import coroutine, http from sugar_network.toolkit.spec import Spec -from sugar_network.client.commands import ClientCommands -from sugar_network.resources.volume import Volume, Resource -from sugar_network.resources.user import User -from sugar_network.resources.context import Context -from sugar_network.resources.implementation import Implementation -from sugar_network.resources.artifact import Artifact +from sugar_network.client.routes import ClientRoutes, Request, Response +from sugar_network.node.master import MasterRoutes +from sugar_network.db import Volume, Resource +from sugar_network.model.user import User +from sugar_network.model.context import Context +from sugar_network.model.implementation import Implementation +from sugar_network.model.artifact import Artifact +from sugar_network.toolkit.router import route import requests -class OnlineCommandsTest(tests.Test): +class OnlineRoutes(tests.Test): def test_inline(self): - cp = ClientCommands(Volume('client'), client.api_url.value) + cp = ClientRoutes(Volume('client', model.RESOURCES), client.api_url.value) assert not cp.inline() trigger = self.wait_for_events(cp, event='inline', state='online') @@ -38,8 +40,8 @@ class OnlineCommandsTest(tests.Test): assert trigger.value is None assert not cp.inline() - request = db.Request(method='GET', cmd='whoami') - cp.call(request) + request = Request(method='GET', cmd='whoami') + cp.whoami(request, Response()) trigger.wait() assert cp.inline() @@ -290,9 +292,9 @@ class OnlineCommandsTest(tests.Test): kwargs['data'] = data.read() updates.append((guid, kwargs)) - self.override(journal.Commands, '__init__', lambda *args: None) - self.override(journal.Commands, 'journal_update', journal_update) - self.override(journal.Commands, 'journal_delete', lambda self, guid: updates.append((guid,))) + self.override(journal.Routes, '__init__', lambda *args: None) + self.override(journal.Routes, 'journal_update', journal_update) + self.override(journal.Routes, 'journal_delete', lambda self, guid: updates.append((guid,))) ipc = IPCClient() @@ -368,9 +370,9 @@ class OnlineCommandsTest(tests.Test): kwargs['data'] = data.read() updates.append((guid, kwargs)) - self.override(journal.Commands, '__init__', lambda *args: None) - self.override(journal.Commands, 'journal_update', journal_update) - self.override(journal.Commands, 'journal_delete', lambda self, guid: updates.append((guid,))) + self.override(journal.Routes, '__init__', lambda *args: None) + self.override(journal.Routes, 'journal_update', journal_update) + self.override(journal.Routes, 'journal_delete', lambda self, guid: updates.append((guid,))) ipc = IPCClient() @@ -497,9 +499,9 @@ class OnlineCommandsTest(tests.Test): job.kill() self.assertEqual([ - {'guid': guid, 'document': 'context', 'event': 'create'}, - {'guid': guid, 'document': 'context', 'event': 'update'}, - {'guid': guid, 'event': 'delete', 'document': 'context'}, + {'guid': guid, 'resource': 'context', 'event': 'create'}, + {'guid': guid, 'resource': 'context', 'event': 'update'}, + {'guid': guid, 'event': 'delete', 'resource': 'context'}, ], events) del events[:] @@ -522,9 +524,9 @@ class OnlineCommandsTest(tests.Test): job.kill() self.assertEqual([ - {'guid': guid, 'document': 'context', 'event': 'create'}, - {'guid': guid, 'document': 'context', 'event': 'update'}, - {'guid': guid, 'event': 'delete', 'document': 'context'}, + {'guid': guid, 'resource': 'context', 'event': 'create'}, + {'guid': guid, 'resource': 'context', 'event': 'update'}, + {'guid': guid, 'event': 'delete', 'resource': 'context'}, ], events) @@ -1097,7 +1099,7 @@ class OnlineCommandsTest(tests.Test): }}) - trigger = self.wait_for_events(ipc, event='update', document='context', guid=context1) + trigger = self.wait_for_events(ipc, event='update', resource='context', guid=context1) ipc.put(['context', context1], 2, cmd='clone') trigger.wait() self.assertEqual( @@ -1110,7 +1112,7 @@ class OnlineCommandsTest(tests.Test): 'summary': 'summary', 'description': 'description', }) - trigger = self.wait_for_events(ipc, event='create', document='context', guid=context2) + trigger = self.wait_for_events(ipc, event='create', resource='context', guid=context2) ipc.put(['context', context2], True, cmd='favorite') trigger.wait() self.assertEqual( @@ -1118,59 +1120,63 @@ class OnlineCommandsTest(tests.Test): ipc.get(['context', context2], reply=['favorite'])) def test_FallbackToLocalSNOnRemoteTransportFails(self): - local_pid = os.getpid() - class Document(Resource): + class LocalRoutes(routes._LocalRoutes): - @db.document_command(method='GET', cmd='sleep') + @route('GET', cmd='sleep') def sleep(self): - if os.getpid() == local_pid: - return 'local' - else: - coroutine.sleep(.5) - return 'remote' + return 'local' - @db.document_command(method='GET', cmd='yield_raw_and_sleep', + @route('GET', cmd='yield_raw_and_sleep', mime_type='application/octet-stream') def yield_raw_and_sleep(self): - if os.getpid() == local_pid: - yield 'local' - else: - for __ in range(33): - yield "remote\n" - coroutine.sleep(.5) - for __ in range(33): - yield "remote\n" - - @db.document_command(method='GET', cmd='yield_json_and_sleep', + yield 'local' + + @route('GET', cmd='yield_json_and_sleep', mime_type='application/json') def yield_json_and_sleep(self): - if os.getpid() == local_pid: - yield '"local"' - else: - yield '"' - yield 'r' - coroutine.sleep(.5) - yield 'emote"' - - home_volume = self.start_client([User, Document]) + yield '"local"' + + self.override(routes, '_LocalRoutes', LocalRoutes) + home_volume = self.start_client([User]) ipc = IPCClient() - guid = ipc.post(['document'], {}) - self.assertEqual('local', ipc.get(['document', guid], cmd='sleep')) - self.assertEqual('local', ipc.get(['document', guid], cmd='yield_raw_and_sleep')) - self.assertEqual('local', ipc.get(['document', guid], cmd='yield_json_and_sleep')) + self.assertEqual('local', ipc.get(cmd='sleep')) + self.assertEqual('local', ipc.get(cmd='yield_raw_and_sleep')) + self.assertEqual('local', ipc.get(cmd='yield_json_and_sleep')) + + class NodeRoutes(MasterRoutes): + + @route('GET', cmd='sleep') + def sleep(self): + coroutine.sleep(.5) + return 'remote' + + @route('GET', cmd='yield_raw_and_sleep', + mime_type='application/octet-stream') + def yield_raw_and_sleep(self): + for __ in range(33): + yield "remote\n" + coroutine.sleep(.5) + for __ in range(33): + yield "remote\n" - node_pid = self.fork_master([User, Document]) + @route('GET', cmd='yield_json_and_sleep', + mime_type='application/json') + def yield_json_and_sleep(self): + yield '"' + yield 'r' + coroutine.sleep(.5) + yield 'emote"' + + node_pid = self.fork_master([User], NodeRoutes) ipc.get(cmd='inline') self.wait_for_events(ipc, event='inline', state='online').wait() - guid = ipc.post(['document'], {}) - home_volume['document'].create({'guid': guid}) ts = time.time() - self.assertEqual('remote', ipc.get(['document', guid], cmd='sleep')) - self.assertEqual('remote\n' * 66, ipc.get(['document', guid], cmd='yield_raw_and_sleep')) - self.assertEqual('remote', ipc.get(['document', guid], cmd='yield_json_and_sleep')) + self.assertEqual('remote', ipc.get(cmd='sleep')) + self.assertEqual('remote\n' * 66, ipc.get(cmd='yield_raw_and_sleep')) + self.assertEqual('remote', ipc.get(cmd='yield_json_and_sleep')) assert time.time() - ts >= 1.5 def kill(): @@ -1178,27 +1184,27 @@ class OnlineCommandsTest(tests.Test): self.waitpid(node_pid) coroutine.spawn(kill) - self.assertEqual('local', ipc.get(['document', guid], cmd='sleep')) + self.assertEqual('local', ipc.get(cmd='sleep')) assert not ipc.get(cmd='inline') - node_pid = self.fork_master([User, Document]) + node_pid = self.fork_master([User], NodeRoutes) ipc.get(cmd='inline') self.wait_for_events(ipc, event='inline', state='online').wait() coroutine.spawn(kill) - self.assertEqual('local', ipc.get(['document', guid], cmd='yield_raw_and_sleep')) + self.assertEqual('local', ipc.get(cmd='yield_raw_and_sleep')) assert not ipc.get(cmd='inline') - node_pid = self.fork_master([User, Document]) + node_pid = self.fork_master([User], NodeRoutes) ipc.get(cmd='inline') self.wait_for_events(ipc, event='inline', state='online').wait() coroutine.spawn(kill) - self.assertEqual('local', ipc.get(['document', guid], cmd='yield_json_and_sleep')) + self.assertEqual('local', ipc.get(cmd='yield_json_and_sleep')) assert not ipc.get(cmd='inline') def test_ReconnectOnServerFall(self): - commands._RECONNECT_TIMEOUT = 1 + routes._RECONNECT_TIMEOUT = 1 node_pid = self.fork_master([User]) self.start_client([User]) diff --git a/tests/units/client/commands.py b/tests/units/client/routes.py index 691eeea..8ae8d44 100755 --- a/tests/units/client/commands.py +++ b/tests/units/client/routes.py @@ -5,25 +5,24 @@ import json from __init__ import tests -from sugar_network import db, client +from sugar_network import db, client, model from sugar_network.client import journal, injector, IPCClient -from sugar_network.client.commands import ClientCommands, CachedClientCommands -from sugar_network.resources.volume import Volume -from sugar_network.resources.user import User -from sugar_network.resources.report import Report -from sugar_network.client import IPCRouter +from sugar_network.client.routes import ClientRoutes, CachedClientRoutes +from sugar_network.model.user import User +from sugar_network.model.report import Report +from sugar_network.toolkit.router import Router, Request, Response from sugar_network.toolkit import coroutine import requests -class CommandsTest(tests.Test): +class RoutesTest(tests.Test): def test_Hub(self): - volume = Volume('db') - cp = ClientCommands(volume) + volume = db.Volume('db', model.RESOURCES) + cp = ClientRoutes(volume) server = coroutine.WSGIServer( - ('127.0.0.1', client.ipc_port.value), IPCRouter(cp)) + ('127.0.0.1', client.ipc_port.value), Router(cp)) coroutine.spawn(server.serve_forever) coroutine.dispatch() @@ -45,13 +44,11 @@ class CommandsTest(tests.Test): def test_launch(self): self.override(injector, 'launch', lambda *args, **kwargs: [{'args': args, 'kwargs': kwargs}]) - volume = Volume('db') - cp = ClientCommands(volume) - - self.assertRaises(RuntimeError, cp.launch, 'fake-document', 'app', []) + volume = db.Volume('db', model.RESOURCES) + cp = ClientRoutes(volume) trigger = self.wait_for_events(cp, event='launch') - cp.launch('context', 'app', []) + cp.launch(Request(path=['context', 'app']), []) self.assertEqual( {'event': 'launch', 'args': ['app', []], 'kwargs': {'color': None, 'activity_id': None, 'uri': None, 'object_id': None}}, trigger.wait()) @@ -59,11 +56,11 @@ class CommandsTest(tests.Test): def test_launch_ResumeJobject(self): self.override(injector, 'launch', lambda *args, **kwargs: [{'args': args, 'kwargs': kwargs}]) self.override(journal, 'exists', lambda *args: True) - volume = Volume('db') - cp = ClientCommands(volume) + volume = db.Volume('db', model.RESOURCES) + cp = ClientRoutes(volume) trigger = self.wait_for_events(cp, event='launch') - cp.launch('context', 'app', [], object_id='object_id') + cp.launch(Request(path=['context', 'app']), [], object_id='object_id') self.assertEqual( {'event': 'launch', 'args': ['app', []], 'kwargs': {'color': None, 'activity_id': None, 'uri': None, 'object_id': 'object_id'}}, trigger.wait()) @@ -165,9 +162,9 @@ class CommandsTest(tests.Test): ipc.get(['context'], reply=['guid', 'title'], favorite=False)['result']) def test_SetLocalLayerInOffline(self): - volume = Volume('client') - cp = ClientCommands(volume, client.api_url.value) - post = db.Request(method='POST', document='context') + volume = db.Volume('client', model.RESOURCES) + cp = ClientRoutes(volume, client.api_url.value) + post = Request(method='POST', path=['context']) post.content_type = 'application/json' post.content = { 'type': 'activity', @@ -176,22 +173,22 @@ class CommandsTest(tests.Test): 'description': 'description', } - guid = cp.call(post) - self.assertEqual(['public', 'local'], cp.call(db.Request(method='GET', document='context', guid=guid, prop='layer'))) + guid = call(cp, post) + self.assertEqual(['public', 'local'], call(cp, Request(method='GET', path=['context', guid, 'layer']))) trigger = self.wait_for_events(cp, event='inline', state='online') node_volume = self.start_master() - cp.call(db.Request(method='GET', cmd='inline')) + call(cp, Request(method='GET', cmd='inline')) trigger.wait() - guid = cp.call(post) - self.assertEqual(['public'], cp.call(db.Request(method='GET', document='context', guid=guid, prop='layer'))) + guid = call(cp, post) + self.assertEqual(['public'], call(cp, Request(method='GET', path=['context', guid, 'layer']))) def test_CachedClientCommands(self): - volume = Volume('client') - cp = CachedClientCommands(volume, client.api_url.value) + volume = db.Volume('client', model.RESOURCES) + cp = CachedClientRoutes(volume, client.api_url.value) - post = db.Request(method='POST', document='context') + post = Request(method='POST', path=['context']) post.content_type = 'application/json' post.content = { 'type': 'activity', @@ -199,12 +196,12 @@ class CommandsTest(tests.Test): 'summary': 'summary', 'description': 'description', } - guid1 = cp.call(post) - guid2 = cp.call(post) + guid1 = call(cp, post) + guid2 = call(cp, post) trigger = self.wait_for_events(cp, event='push') self.start_master() - cp.call(db.Request(method='GET', cmd='inline')) + call(cp, Request(method='GET', cmd='inline')) trigger.wait() self.assertEqual([[3, None]], json.load(file('client/push.sequence'))) @@ -229,7 +226,7 @@ class CommandsTest(tests.Test): trigger = self.wait_for_events(cp, event='push') self.start_master() - cp.call(db.Request(method='GET', cmd='inline')) + call(cp, Request(method='GET', cmd='inline')) trigger.wait() self.assertEqual([[4, None]], json.load(file('client/push.sequence'))) @@ -245,30 +242,30 @@ class CommandsTest(tests.Test): self.node_volume['context'].get(guid2)['author']) def test_CachedClientCommands_WipeReports(self): - volume = Volume('client') - cp = CachedClientCommands(volume, client.api_url.value) + volume = db.Volume('client', model.RESOURCES) + cp = CachedClientRoutes(volume, client.api_url.value) - post = db.Request(method='POST', document='report') + post = Request(method='POST', path=['report']) post.content_type = 'application/json' post.content = { 'context': 'context', 'error': 'error', } - guid = cp.call(post) + guid = call(cp, post) trigger = self.wait_for_events(cp, event='push') self.start_master([User, Report]) - cp.call(db.Request(method='GET', cmd='inline')) + call(cp, Request(method='GET', cmd='inline')) trigger.wait() assert not volume['report'].exists(guid) assert self.node_volume['report'].exists(guid) def test_SwitchToOfflineForAbsentOnlineProps(self): - volume = Volume('client') - cp = ClientCommands(volume, client.api_url.value) + volume = db.Volume('client', model.RESOURCES) + cp = ClientRoutes(volume, client.api_url.value) - post = db.Request(method='POST', document='context') + post = Request(method='POST', path=['context']) post.content_type = 'application/json' post.content = { 'type': 'activity', @@ -276,17 +273,22 @@ class CommandsTest(tests.Test): 'summary': 'summary', 'description': 'description', } - guid = cp.call(post) + guid = call(cp, post) - self.assertEqual('title', cp.call(db.Request(method='GET', document='context', guid=guid, prop='title'))) + self.assertEqual('title', call(cp, Request(method='GET', path=['context', guid, 'title']))) trigger = self.wait_for_events(cp, event='inline', state='online') self.start_master() - cp.call(db.Request(method='GET', cmd='inline')) + call(cp, Request(method='GET', cmd='inline')) trigger.wait() assert not self.node_volume['context'].exists(guid) - self.assertEqual('title', cp.call(db.Request(method='GET', document='context', guid=guid, prop='title'))) + self.assertEqual('title', call(cp, Request(method='GET', path=['context', guid, 'title']))) + + +def call(routes, request): + router = Router(routes) + return router.call(request, Response()) if __name__ == '__main__': diff --git a/tests/units/client/server_commands.py b/tests/units/client/server_routes.py index 771d008..bea8f7f 100755 --- a/tests/units/client/server_commands.py +++ b/tests/units/client/server_routes.py @@ -7,11 +7,11 @@ from os.path import exists from __init__ import tests, src_root -from sugar_network import db, client +from sugar_network import db, client, model from sugar_network.client import IPCClient -from sugar_network.client.commands import ClientCommands -from sugar_network.client import IPCRouter -from sugar_network.resources.volume import Volume +from sugar_network.client.routes import ClientRoutes +from sugar_network.db import Volume +from sugar_network.toolkit.router import Router from sugar_network.toolkit import mountpoints, coroutine @@ -19,20 +19,20 @@ class ServerCommandsTest(tests.Test): def start_node(self): os.makedirs('disk/sugar-network') - self.node_volume = Volume('db') - cp = ClientCommands(self.node_volume) + self.node_volume = Volume('db', model.RESOURCES) + cp = ClientRoutes(self.node_volume) trigger = self.wait_for_events(cp, event='inline', state='online') coroutine.spawn(mountpoints.monitor, tests.tmpdir) trigger.wait() - server = coroutine.WSGIServer(('127.0.0.1', client.ipc_port.value), IPCRouter(cp)) + server = coroutine.WSGIServer(('127.0.0.1', client.ipc_port.value), Router(cp)) coroutine.spawn(server.serve_forever) coroutine.dispatch() return cp def test_PopulateNode(self): os.makedirs('disk/sugar-network') - volume = Volume('db') - cp = ClientCommands(volume) + volume = Volume('db', model.RESOURCES) + cp = ClientRoutes(volume) assert not cp.inline() trigger = self.wait_for_events(cp, event='inline', state='online') @@ -41,8 +41,8 @@ class ServerCommandsTest(tests.Test): assert cp.inline() def test_MountNode(self): - volume = Volume('db') - cp = ClientCommands(volume) + volume = Volume('db', model.RESOURCES) + cp = ClientRoutes(volume) trigger = self.wait_for_events(cp, event='inline', state='online') mountpoints.populate('.') @@ -97,8 +97,8 @@ class ServerCommandsTest(tests.Test): job.kill() self.assertEqual([ - {'guid': guid, 'document': 'context', 'event': 'create'}, - {'guid': guid, 'document': 'context', 'event': 'update'}, + {'guid': guid, 'resource': 'context', 'event': 'create'}, + {'guid': guid, 'resource': 'context', 'event': 'update'}, ], events) diff --git a/tests/units/db/__main__.py b/tests/units/db/__main__.py index 70e829a..6dda7ff 100644 --- a/tests/units/db/__main__.py +++ b/tests/units/db/__main__.py @@ -2,15 +2,11 @@ from __init__ import tests -from commands import * -from document import * -from env import * +from resource import * from index import * -from metadata import * from migrate import * -from router import * from storage import * -from volume import * +from routes import * if __name__ == '__main__': tests.main() diff --git a/tests/units/db/commands.py b/tests/units/db/commands.py deleted file mode 100755 index da859d5..0000000 --- a/tests/units/db/commands.py +++ /dev/null @@ -1,552 +0,0 @@ -#!/usr/bin/env python -# sugar-lint: disable - -from cStringIO import StringIO - -from __init__ import tests - -from sugar_network import db -from sugar_network.db import env, volume, Volume, Document, \ - property_command, document_command, directory_command, volume_command, \ - Request, BlobProperty, Response, CommandsProcessor, \ - CommandNotFound, to_int, to_list -from sugar_network.db.router import route -from sugar_network.toolkit.http import NotFound, Forbidden - - -class CommandsTest(tests.Test): - - def test_VolumeCommands(self): - calls = [] - - class TestCommandsProcessor(CommandsProcessor): - - @volume_command(method='PROBE') - def command_1(self, **kwargs): - calls.append(('command_1', kwargs)) - - @volume_command(method='PROBE', cmd='command_2') - def command_2(self, **kwargs): - calls.append(('command_2', kwargs)) - - cp = TestCommandsProcessor() - - self.call(cp, 'PROBE') - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', cmd='command_1') - self.call(cp, 'PROBE', cmd='command_2') - - self.assertEqual([ - ('command_1', {}), - ('command_2', {}), - ], - calls) - - def test_DirectoryCommands(self): - calls = [] - - class TestCommandsProcessor(CommandsProcessor): - - @directory_command(method='PROBE') - def command_1(self, **kwargs): - calls.append(('command_1', kwargs)) - - @directory_command(method='PROBE', cmd='command_2') - def command_2(self, **kwargs): - calls.append(('command_2', kwargs)) - - cp = TestCommandsProcessor() - - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE') - self.call(cp, 'PROBE', document='testdocument') - self.call(cp, 'PROBE', document='fakedocument') - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', cmd='command_1', document='testdocument') - self.call(cp, 'PROBE', cmd='command_2', document='testdocument') - self.call(cp, 'PROBE', cmd='command_2', document='fakedocument') - - self.assertEqual([ - ('command_1', {}), - ('command_1', {}), - ('command_2', {}), - ('command_2', {}), - ], - calls) - - def test_DocumentCommands(self): - calls = [] - - class TestCommandsProcessor(CommandsProcessor): - - @document_command(method='PROBE') - def command_1(self, **kwargs): - calls.append(('command_1', kwargs)) - - @document_command(method='PROBE', cmd='command_2') - def command_2(self, **kwargs): - calls.append(('command_2', kwargs)) - - class TestDocument(Document): - pass - - volume = Volume(tests.tmpdir, [TestDocument]) - cp = TestCommandsProcessor(volume) - - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE') - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', document='testdocument') - self.call(cp, 'PROBE', document='testdocument', guid='guid') - self.call(cp, 'PROBE', document='fakedocument', guid='guid') - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', cmd='command_1', document='testdocument', guid='guid') - self.call(cp, 'PROBE', cmd='command_2', document='testdocument', guid='guid') - self.call(cp, 'PROBE', cmd='command_2', document='fakedocument', guid='guid') - - self.assertEqual([ - ('command_1', {}), - ('command_1', {}), - ('command_2', {}), - ('command_2', {}), - ], - calls) - - def test_PropertyCommands(self): - calls = [] - - class TestCommandsProcessor(CommandsProcessor): - - @property_command(method='PROBE') - def command_1(self, **kwargs): - calls.append(('command_1', kwargs)) - - @property_command(method='PROBE', cmd='command_2') - def command_2(self, **kwargs): - calls.append(('command_2', kwargs)) - - class TestDocument(Document): - pass - - volume = Volume(tests.tmpdir, [TestDocument]) - cp = TestCommandsProcessor(volume) - - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE') - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', document='testdocument') - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', document='testdocument', guid='guid') - self.call(cp, 'PROBE', document='testdocument', guid='guid', prop='prop') - self.call(cp, 'PROBE', document='fakedocument', guid='guid', prop='prop') - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', cmd='command_1', document='testdocument', guid='guid', prop='prop') - self.call(cp, 'PROBE', cmd='command_2', document='testdocument', guid='guid', prop='prop') - self.call(cp, 'PROBE', cmd='command_2', document='fakedocument', guid='guid', prop='prop') - - self.assertEqual([ - ('command_1', {}), - ('command_1', {}), - ('command_2', {}), - ('command_2', {}), - ], - calls) - - def test_ClassDodcumentCommands(self): - calls = [] - - class TestDocument(Document): - - @document_command(method='PROBE') - def command_1(cls, **kwargs): - calls.append(('command_1', kwargs)) - - @document_command(method='PROBE', cmd='command_2') - def command_2(cls, **kwargs): - calls.append(('command_2', kwargs)) - - volume = Volume(tests.tmpdir, [TestDocument]) - cp = CommandsProcessor(volume) - - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE') - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', document='testdocument') - self.assertRaises(NotFound, self.call, cp, 'PROBE', document='testdocument', guid='guid') - volume['testdocument'].create({'guid': 'guid'}) - self.call(cp, 'PROBE', document='testdocument', guid='guid') - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', document='fakedocument', guid='guid') - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', cmd='command_1', document='testdocument', guid='guid') - self.call(cp, 'PROBE', cmd='command_2', document='testdocument', guid='guid') - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', cmd='command_2', document='fakedocument', guid='guid') - - self.assertEqual([ - ('command_1', {}), - ('command_2', {}), - ], - calls) - - def test_ClassPropertyCommands(self): - calls = [] - - class TestDocument(Document): - - @property_command(method='PROBE') - def command_1(cls, **kwargs): - calls.append(('command_1', kwargs)) - - @property_command(method='PROBE', cmd='command_2') - def command_2(cls, **kwargs): - calls.append(('command_2', kwargs)) - - volume = Volume(tests.tmpdir, [TestDocument]) - cp = CommandsProcessor(volume) - - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE') - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', document='testdocument') - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', document='testdocument', prop='prop') - self.assertRaises(NotFound, self.call, cp, 'PROBE', document='testdocument', guid='guid', prop='prop') - volume['testdocument'].create({'guid': 'guid'}) - self.call(cp, 'PROBE', document='testdocument', guid='guid', prop='prop') - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', document='fakedocument', guid='guid', prop='prop') - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', cmd='command_1', document='testdocument', guid='guid', prop='prop') - self.call(cp, 'PROBE', cmd='command_2', document='testdocument', guid='guid', prop='prop') - self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', cmd='command_2', document='fakedocument', guid='guid', prop='prop') - - self.assertEqual([ - ('command_1', {}), - ('command_2', {}), - ], - calls) - - def test_AccessLevel(self): - calls = [] - - class TestCommandsProcessor(CommandsProcessor): - - @volume_command(method='PROBE', cmd='all') - def all(self): - pass - - @volume_command(method='PROBE', cmd='system', access_level=env.ACCESS_SYSTEM) - def system(self): - pass - - @volume_command(method='PROBE', cmd='local', access_level=env.ACCESS_LOCAL) - def local(self): - pass - - @volume_command(method='PROBE', cmd='remote', access_level=env.ACCESS_REMOTE) - def remote(self): - pass - - cp = TestCommandsProcessor() - - self.call(cp, 'PROBE', cmd='all', access_level=env.ACCESS_REMOTE) - self.call(cp, 'PROBE', cmd='all', access_level=env.ACCESS_LOCAL) - self.call(cp, 'PROBE', cmd='all', access_level=env.ACCESS_SYSTEM) - - self.call(cp, 'PROBE', cmd='remote', access_level=env.ACCESS_REMOTE) - self.assertRaises(Forbidden, self.call, cp, 'PROBE', cmd='remote', access_level=env.ACCESS_LOCAL) - self.assertRaises(Forbidden, self.call, cp, 'PROBE', cmd='remote', access_level=env.ACCESS_SYSTEM) - - self.assertRaises(Forbidden, self.call, cp, 'PROBE', cmd='local', access_level=env.ACCESS_REMOTE) - self.call(cp, 'PROBE', cmd='local', access_level=env.ACCESS_LOCAL) - self.assertRaises(Forbidden, self.call, cp, 'PROBE', cmd='local', access_level=env.ACCESS_SYSTEM) - - self.assertRaises(Forbidden, self.call, cp, 'PROBE', cmd='system', access_level=env.ACCESS_REMOTE) - self.assertRaises(Forbidden, self.call, cp, 'PROBE', cmd='system', access_level=env.ACCESS_LOCAL) - self.call(cp, 'PROBE', cmd='system', access_level=env.ACCESS_SYSTEM) - - def test_ParentClasses(self): - calls = [] - - class Parent(object): - - @volume_command(method='PROBE') - def probe(self): - return 'probe' - - class TestCommandsProcessor(CommandsProcessor, Parent): - pass - - cp = TestCommandsProcessor() - self.assertEqual('probe', self.call(cp, 'PROBE')) - - def test_OverrideInChildClass(self): - calls = [] - - class Parent(CommandsProcessor): - - @volume_command(method='PROBE') - def probe(self): - return 'probe-1' - - @volume_command(method='COMMON') - def common(self): - return 'common' - - class Child(Parent): - - @volume_command(method='PROBE') - def probe(self): - return 'probe-2' - - @volume_command(method='PARTICULAR') - def particular(self): - return 'particular' - - cp = Child() - self.assertEqual('probe-2', self.call(cp, 'PROBE')) - self.assertEqual('common', self.call(cp, 'COMMON')) - self.assertEqual('particular', self.call(cp, 'PARTICULAR')) - - def test_RequestRead(self): - - class Stream(object): - - def __init__(self, value): - self.pos = 0 - self.value = value - - def read(self, size): - assert self.pos + size <= len(self.value) - result = self.value[self.pos:self.pos + size] - self.pos += size - return result - - request = Request() - request.content_stream = Stream('123') - request.content_length = len(request.content_stream.value) - self.assertEqual('123', request.read()) - self.assertEqual('', request.read()) - self.assertEqual('', request.read(10)) - - request = Request() - request.content_stream = Stream('123') - request.content_length = len(request.content_stream.value) - self.assertEqual('123', request.read(10)) - - request = Request() - request.content_stream = Stream('123') - request.content_length = len(request.content_stream.value) - self.assertEqual('1', request.read(1)) - self.assertEqual('2', request.read(1)) - self.assertEqual('3', request.read()) - - def test_Arguments(self): - - class TestCommandsProcessor(CommandsProcessor): - - @volume_command(method='PROBE', arguments={'arg_int': to_int, 'arg_list': to_list}) - def probe(self, arg_int=None, arg_list=None): - return arg_int, arg_list - - cp = TestCommandsProcessor() - - self.assertEqual((None, None), self.call(cp, 'PROBE')) - self.assertEqual((-1, [-2, None]), self.call(cp, 'PROBE', arg_int=-1, arg_list=[-2, None])) - self.assertEqual((4, [' foo', ' bar ', ' ']), self.call(cp, 'PROBE', arg_int='4', arg_list=' foo, bar , ')) - self.assertEqual((None, ['foo']), self.call(cp, 'PROBE', arg_list='foo')) - self.assertEqual((None, []), self.call(cp, 'PROBE', arg_list='')) - self.assertEqual((None, [' ']), self.call(cp, 'PROBE', arg_list=' ')) - self.assertEqual((0, None), self.call(cp, 'PROBE', arg_int='')) - self.assertRaises(RuntimeError, self.call, cp, 'PROBE', arg_int=' ') - self.assertRaises(RuntimeError, self.call, cp, 'PROBE', arg_int='foo') - - def test_PassKwargs(self): - - class TestCommandsProcessor(CommandsProcessor): - - @volume_command(method='PROBE') - def probe(self, arg, request, response, **kwargs): - return arg, dict(request), dict(response), kwargs - - cp = TestCommandsProcessor() - - self.assertEqual( - (None, {'method': 'PROBE'}, {}, {}), - self.call(cp, 'PROBE')) - self.assertEqual( - (1, {'method': 'PROBE', 'arg': 1}, {}, {}), - self.call(cp, 'PROBE', arg=1)) - self.assertEqual( - (None, {'method': 'PROBE', 'foo': 'bar'}, {}, {}), - self.call(cp, 'PROBE', foo='bar')) - self.assertEqual( - (-2, {'method': 'PROBE', 'foo': 'bar', 'arg': -2}, {}, {}), - self.call(cp, 'PROBE', foo='bar', arg=-2)) - - def test_PrePost(self): - - class ParentCommandsProcessor(CommandsProcessor): - - @db.volume_command_pre(method='PROBE') - def command_pre1(self, request): - request['probe'].append('pre1') - - @db.volume_command_pre(method='PROBE') - def command_pre2(self, request): - request['probe'].append('pre2') - - @db.volume_command_post(method='PROBE') - def command_post1(self, request, response, result): - request['probe'].append('post1') - response['probe'].append('post1') - return result + 1 - - @db.volume_command_post(method='PROBE') - def command_post2(self, request, response, result): - request['probe'].append('post2') - response['probe'].append('post2') - return result + 1 - - class TestCommandsProcessor(ParentCommandsProcessor): - - @db.volume_command_pre(method='PROBE') - def command_pre3(self, request): - request['probe'].append('pre3') - - @db.volume_command_pre(method='PROBE') - def command_pre4(self, request): - request['probe'].append('pre4') - - @db.volume_command(method='PROBE') - def command(self, request): - request['probe'].append('cmd') - response['probe'].append('cmd') - return 1 - - @db.volume_command_post(method='PROBE') - def command_post3(self, request, response, result): - request['probe'].append('post3') - response['probe'].append('post3') - return result + 1 - - @db.volume_command_post(method='PROBE') - def command_post4(self, request, response, result): - request['probe'].append('post4') - response['probe'].append('post4') - return result + 1 - - cp = TestCommandsProcessor() - - request = db.Request(method='PROBE', probe=[]) - response = db.Response(probe=[]) - self.assertEqual(5, cp.call(request, response)) - self.assertEqual(['pre1', 'pre2', 'pre3', 'pre4', 'cmd', 'post1', 'post2', 'post3', 'post4'], request['probe']) - self.assertEqual(['cmd', 'post1', 'post2', 'post3', 'post4'], response['probe']) - - def test_PrePostCallbackLess(self): - - class TestCommandsProcessor(CommandsProcessor): - - @db.volume_command_pre(method='PROBE') - def command_pre(self, request): - request['probe'].append('pre') - - def super_call(self, request, response): - request['probe'].append('cmd') - response['probe'].append('cmd') - return 1 - - @db.volume_command_post(method='PROBE') - def command_post(self, request, response, result): - request['probe'].append('post') - response['probe'].append('post') - return result + 1 - - cp = TestCommandsProcessor() - - request = db.Request(method='PROBE', probe=[]) - response = db.Response(probe=[]) - self.assertEqual(2, cp.call(request, response)) - self.assertEqual(['pre', 'cmd', 'post'], request['probe']) - self.assertEqual(['cmd', 'post'], response['probe']) - - def test_SubCall(self): - - class TestCommandsProcessor(CommandsProcessor): - - @db.volume_command(method='PROBE') - def command1(self, request): - return request.call('PROBE', cmd='command2') - - @db.volume_command(method='PROBE') - def command2(self, request): - return {'access_level': request.access_level, 'accept_language': request.accept_language} - - cp = TestCommandsProcessor() - - request = db.Request(method='PROBE') - request.access_level = -1 - request.accept_language = 'foo' - self.assertEqual({ - 'access_level': -1, - 'accept_language': 'foo', - }, - cp.call(request, db.Response())) - - def test_Routes(self): - calls = [] - - class BaseRoutes(CommandsProcessor): - - @route('GET', '/foo') - def route1(self, request, response): - return 'route1' - - class Routes(BaseRoutes): - - @route('PUT', '/foo') - def route2(self, request, response): - return 'route2' - - @route('GET', '/bar') - def route3(self, request, response): - return 'route3' - - def call(self, request, response): - try: - return CommandsProcessor.call(self, request, response) - except CommandNotFound: - return 'default' - - cp = Routes() - request = Request() - - request.path = [] - request['method'] = 'GET' - self.assertEqual('default', cp.call(request, db.Response())) - - request.path = ['foo'] - request['method'] = 'GET' - self.assertEqual('route1', cp.call(request, db.Response())) - - request.path = ['foo'] - request['method'] = 'PUT' - self.assertEqual('route2', cp.call(request, db.Response())) - - request.path = ['foo'] - request['method'] = 'POST' - self.assertEqual('default', cp.call(request, db.Response())) - - request.path = ['bar', 'foo', 'probe'] - request['method'] = 'GET' - self.assertEqual('route3', cp.call(request, db.Response())) - - def call(self, cp, method, document=None, guid=None, prop=None, - access_level=env.ACCESS_REMOTE, **kwargs): - - class TestRequest(Request): - - content_stream = None - content_length = 0 - - request = TestRequest(**kwargs) - request['method'] = method - request.access_level = access_level - if document: - request['document'] = document - if guid: - request['guid'] = guid - if prop: - request['prop'] = prop - if 'content_stream' in request: - request.content_stream = request.pop('content_stream') - request.content_length = len(request.content_stream.getvalue()) - - self.response = Response() - return cp.call(request, self.response) - - -if __name__ == '__main__': - tests.main() diff --git a/tests/units/db/env.py b/tests/units/db/env.py deleted file mode 100755 index 953271e..0000000 --- a/tests/units/db/env.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -# sugar-lint: disable - -import copy -from os.path import exists - -from __init__ import tests - -from sugar_network import toolkit -from sugar_network.db import env - - -class EnvTest(tests.Test): - - def test_gettext(self): - # Fallback to default lang - toolkit._default_lang = 'default' - self.assertEqual('foo', env.gettext({'lang': 'foo', 'default': 'bar'}, 'lang')) - self.assertEqual('bar', env.gettext({'lang': 'foo', 'default': 'bar'}, 'fake')) - - # Exact accept_language - self.assertEqual('', env.gettext(None, 'lang')) - self.assertEqual('foo', env.gettext('foo', 'lang')) - self.assertEqual('foo', env.gettext({'lang': 'foo', 'fake': 'bar', 'default': 'default'}, 'lang')) - self.assertEqual('foo', env.gettext({'lang': 'foo', 'fake': 'bar', 'default': 'default'}, ['lang', 'fake'])) - self.assertEqual('bar', env.gettext({'lang': 'foo', 'fake': 'bar', 'default': 'default'}, ['fake', 'lang'])) - - # Last resort - self.assertEqual('foo', env.gettext({'1': 'foo', '2': 'bar'}, 'fake')) - - # Primed accept_language - self.assertEqual('foo', env.gettext({'1': 'foo', '2': 'bar', 'default': 'default'}, '1-a')) - - # Primed i18n value - self.assertEqual('bar', env.gettext({'1-a': 'foo', '1': 'bar', 'default': 'default'}, '1-b')) - self.assertEqual('foo', env.gettext({'1-a': 'foo', '2': 'bar', 'default': 'default'}, '1-b')) - - def test_gettext_EnAsTheLastResort(self): - toolkit._default_lang = 'en-us' - self.assertEqual('right', env.gettext({'a': 'wrong', 'en': 'right'}, 'probe')) - self.assertEqual('exact', env.gettext({'a': 'wrong', 'en': 'right', 'probe': 'exact'}, 'probe')) - - -if __name__ == '__main__': - tests.main() diff --git a/tests/units/db/index.py b/tests/units/db/index.py index b94675a..9b7d130 100755 --- a/tests/units/db/index.py +++ b/tests/units/db/index.py @@ -11,9 +11,11 @@ from os.path import exists from __init__ import tests from sugar_network import toolkit -from sugar_network.db import index, env -from sugar_network.db.metadata import Metadata, IndexedProperty, GUID_PREFIX +from sugar_network.db import index +from sugar_network.db.index import _fmt_prop_value +from sugar_network.db.metadata import Metadata, IndexedProperty, GUID_PREFIX, Property from sugar_network.db.directory import _Query +from sugar_network.toolkit.router import ACL from sugar_network.toolkit import coroutine @@ -69,8 +71,8 @@ class IndexTest(tests.Test): ([], 0), db._find(reply=['key'])) - def test_IndexByReprcast(self): - db = Index({'key': IndexedProperty('key', 1, 'K', reprcast=lambda x: "foo" + x)}) + def test_IndexByFmt(self): + db = Index({'key': IndexedProperty('key', 1, 'K', fmt=lambda x: "foo" + x)}) db.store('1', {'key': 'bar'}) @@ -87,7 +89,7 @@ class IndexTest(tests.Test): [], db._find(key='fake', reply=['key'])[0]) - def test_IndexByReprcastGenerator(self): + def test_IndexByFmtGenerator(self): def iterate(value): if value != 'fake': @@ -95,7 +97,7 @@ class IndexTest(tests.Test): yield 'bar' yield value - db = Index({'key': IndexedProperty('key', 1, 'K', reprcast=iterate)}) + db = Index({'key': IndexedProperty('key', 1, 'K', fmt=iterate)}) db.store('1', {'key': 'value'}) self.assertEqual( @@ -497,7 +499,7 @@ class IndexTest(tests.Test): db = Index({}, lambda: commits.append(True)) coroutine.dispatch() - env.index_flush_threshold.value = 1 + index.index_flush_threshold.value = 1 db.store('1', {}) coroutine.dispatch() db.store('2', {}) @@ -510,7 +512,7 @@ class IndexTest(tests.Test): del commits[:] db = Index({}, lambda: commits.append(True)) coroutine.dispatch() - env.index_flush_threshold.value = 2 + index.index_flush_threshold.value = 2 db.store('4', {}) coroutine.dispatch() db.store('5', {}) @@ -525,8 +527,8 @@ class IndexTest(tests.Test): db.close() def test_FlushTimeout(self): - env.index_flush_threshold.value = 0 - env.index_flush_timeout.value = 1 + index.index_flush_threshold.value = 0 + index.index_flush_timeout.value = 1 commits = [] @@ -557,7 +559,7 @@ class IndexTest(tests.Test): self.assertEqual(2, len(commits)) def test_DoNotMissImmediateCommitEvent(self): - env.index_flush_threshold.value = 1 + index.index_flush_threshold.value = 1 commits = [] db = Index({}, lambda: commits.append(True)) @@ -748,6 +750,25 @@ class IndexTest(tests.Test): ]), db._find(prop=['a', '-b', 'c'], reply=['guid'])[0]) + def test_fmt_prop_value(self): + prop = Property('prop') + self.assertEqual(['0'], [i for i in _fmt_prop_value(prop, 0)]) + self.assertEqual(['1'], [i for i in _fmt_prop_value(prop, 1)]) + self.assertEqual(['0'], [i for i in _fmt_prop_value(prop, 0)]) + self.assertEqual(['1.1'], [i for i in _fmt_prop_value(prop, 1.1)]) + self.assertEqual(['0', '1'], [i for i in _fmt_prop_value(prop, [0, 1])]) + self.assertEqual(['2', '1'], [i for i in _fmt_prop_value(prop, [2, 1])]) + self.assertEqual(['probe', 'True', '0'], [i for i in _fmt_prop_value(prop, ['probe', True, 0])]) + self.assertEqual(['True'], [i for i in _fmt_prop_value(prop, True)]) + self.assertEqual(['False'], [i for i in _fmt_prop_value(prop, False)]) + + prop = Property('prop', typecast=bool) + self.assertEqual(['1'], [i for i in _fmt_prop_value(prop, True)]) + self.assertEqual(['0'], [i for i in _fmt_prop_value(prop, False)]) + + prop = Property('prop', fmt=lambda x: x.keys()) + self.assertEqual(['a', '2'], [i for i in _fmt_prop_value(prop, {'a': 1, 2: 'b'})]) + class Index(index.IndexWriter): @@ -759,7 +780,7 @@ class Index(index.IndexWriter): metadata = Metadata(Index) metadata.update(props) metadata['guid'] = IndexedProperty('guid', - permissions=env.ACCESS_CREATE | env.ACCESS_READ, slot=0, + acl=ACL.CREATE | ACL.READ, slot=0, prefix=GUID_PREFIX) index.IndexWriter.__init__(self, tests.tmpdir + '/index', metadata, *args) diff --git a/tests/units/db/metadata.py b/tests/units/db/metadata.py deleted file mode 100755 index 64c08db..0000000 --- a/tests/units/db/metadata.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python -# sugar-lint: disable - -from __init__ import tests - -from sugar_network.db.metadata import Property - - -class MetadataTest(tests.Test): - - def test_Property_decode(self): - prop = Property('prop', typecast=int) - self.assertEqual(1, prop.decode(1)) - self.assertEqual(1, prop.decode(1.1)) - self.assertEqual(1, prop.decode('1')) - self.assertRaises(ValueError, prop.decode, '1.0') - self.assertRaises(ValueError, prop.decode, '') - self.assertRaises(ValueError, prop.decode, None) - - prop = Property('prop', typecast=float) - self.assertEqual(1.0, prop.decode(1)) - self.assertEqual(1.1, prop.decode(1.1)) - self.assertEqual(1.0, prop.decode('1')) - self.assertEqual(1.1, prop.decode('1.1')) - self.assertRaises(ValueError, prop.decode, '') - self.assertRaises(ValueError, prop.decode, None) - - prop = Property('prop', typecast=bool) - self.assertEqual(False, prop.decode(0)) - self.assertEqual(True, prop.decode(1)) - self.assertEqual(True, prop.decode(1.1)) - self.assertEqual(True, prop.decode('1')) - self.assertEqual(True, prop.decode('A')) - self.assertEqual(False, prop.decode('')) - self.assertRaises(ValueError, prop.decode, None) - - prop = Property('prop', typecast=[int]) - self.assertEqual((1,), prop.decode(1)) - self.assertRaises(ValueError, prop.decode, None) - self.assertRaises(ValueError, prop.decode, '') - self.assertEqual((), prop.decode([])) - self.assertEqual((123,), prop.decode('123')) - self.assertRaises(ValueError, prop.decode, 'a') - self.assertEqual((123, 4, 5), prop.decode(['123', 4, 5.6])) - - prop = Property('prop', typecast=[1, 2]) - self.assertRaises(ValueError, prop.decode, 0) - self.assertRaises(ValueError, prop.decode, None) - self.assertRaises(ValueError, prop.decode, '') - self.assertRaises(ValueError, prop.decode, 'A') - self.assertEqual(1, prop.decode(1)) - self.assertEqual(2, prop.decode(2)) - - prop = Property('prop', typecast=[[True, False, 'probe']]) - self.assertRaises(ValueError, prop.decode, None) - self.assertEqual((0, ), prop.decode(0)) - self.assertRaises(ValueError, prop.decode, 'A') - self.assertEqual((True, ), prop.decode(True)) - self.assertEqual((False, ), prop.decode(False)) - self.assertRaises(ValueError, prop.decode, [3]) - self.assertRaises(ValueError, prop.decode, ['A']) - self.assertRaises(ValueError, prop.decode, '') - self.assertEqual((), prop.decode([])) - self.assertEqual((True,), prop.decode([True])) - self.assertEqual((False,), prop.decode([False])) - self.assertEqual((True, False, True), prop.decode([True, False, True])) - self.assertEqual((True, False, 'probe'), prop.decode([True, False, 'probe'])) - self.assertRaises(ValueError, prop.decode, [True, None]) - - prop = Property('prop', typecast=[str]) - self.assertEqual(('',), prop.decode('')) - self.assertEqual(('',), prop.decode([''])) - self.assertEqual((), prop.decode([])) - - prop = Property('prop', typecast=[]) - self.assertRaises(ValueError, prop.decode, None) - self.assertEqual(('',), prop.decode('')) - self.assertEqual(('',), prop.decode([''])) - self.assertEqual((), prop.decode([])) - self.assertEqual(('0',), prop.decode(0)) - self.assertEqual(('',), prop.decode('')) - self.assertEqual(('foo',), prop.decode('foo')) - - prop = Property('prop', typecast=[['A', 'B', 'C']]) - self.assertRaises(ValueError, prop.decode, '') - self.assertRaises(ValueError, prop.decode, ['']) - self.assertEqual((), prop.decode([])) - self.assertEqual(('A', 'B', 'C'), prop.decode(['A', 'B', 'C'])) - self.assertRaises(ValueError, prop.decode, ['a']) - self.assertRaises(ValueError, prop.decode, ['A', 'x']) - - prop = Property('prop', typecast=[frozenset(['A', 'B', 'C'])]) - self.assertEqual(('A', 'B', 'C'), prop.decode(['A', 'B', 'C'])) - - prop = Property('prop', typecast=lambda x: x + 1) - self.assertEqual(1, prop.decode(0)) - - def test_Property_to_string(self): - prop = Property('prop', typecast=int) - self.assertEqual(['0'], prop.to_string(0)) - self.assertEqual(['1'], prop.to_string(1)) - - prop = Property('prop', typecast=float) - self.assertEqual(['0'], prop.to_string(0)) - self.assertEqual(['1.1'], prop.to_string(1.1)) - - prop = Property('prop', typecast=bool) - self.assertEqual(['1'], prop.to_string(True)) - self.assertEqual(['0'], prop.to_string(False)) - - prop = Property('prop', typecast=[int]) - self.assertEqual(['0', '1'], prop.to_string([0, 1])) - - prop = Property('prop', typecast=[1, 2]) - self.assertEqual(['2', '1'], prop.to_string([2, 1])) - - prop = Property('prop', typecast=[[True, 0, 'probe']]) - self.assertEqual(['probe', '1', '0'], prop.to_string(['probe', True, 0])) - - prop = Property('prop', reprcast=lambda x: x.keys()) - self.assertEqual(['a', '2'], prop.to_string({'a': 1, 2: 'b'})) - - -if __name__ == '__main__': - tests.main() diff --git a/tests/units/db/migrate.py b/tests/units/db/migrate.py index 89e75c8..773a45d 100755 --- a/tests/units/db/migrate.py +++ b/tests/units/db/migrate.py @@ -8,7 +8,6 @@ from os.path import exists, lexists from __init__ import tests from sugar_network import db -from sugar_network.db import document, env from sugar_network.db import directory as directory_ from sugar_network.db.directory import Directory from sugar_network.db.index import IndexWriter @@ -18,7 +17,7 @@ class MigrateTest(tests.Test): def test_MissedProps(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(prefix='P') def prop1(self, value): @@ -52,7 +51,7 @@ class MigrateTest(tests.Test): def test_ConvertToJson(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(prefix='P', default='value') def prop(self, value): diff --git a/tests/units/db/document.py b/tests/units/db/resource.py index e25ee29..ed37664 100755 --- a/tests/units/db/document.py +++ b/tests/units/db/resource.py @@ -18,18 +18,19 @@ import gobject from __init__ import tests from sugar_network import db -from sugar_network.db import document, storage, env, index +from sugar_network.db import storage, index from sugar_network.db import directory as directory_ from sugar_network.db.directory import Directory from sugar_network.db.index import IndexWriter -from sugar_network.toolkit.util import Sequence +from sugar_network.toolkit.router import ACL +from sugar_network.toolkit import Sequence -class DocumentTest(tests.Test): +class ResourceTest(tests.Test): def test_ActiveProperty_Slotted(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def slotted(self, value): @@ -54,7 +55,7 @@ class DocumentTest(tests.Test): def test_ActiveProperty_SlottedIUnique(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def prop_1(self, value): @@ -68,7 +69,7 @@ class DocumentTest(tests.Test): def test_ActiveProperty_Terms(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(prefix='T') def term(self, value): @@ -94,7 +95,7 @@ class DocumentTest(tests.Test): def test_ActiveProperty_TermsUnique(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(prefix='P') def prop_1(self, value): @@ -108,7 +109,7 @@ class DocumentTest(tests.Test): def test_ActiveProperty_FullTextSearch(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(full_text=False, slot=1) def no(self, value): @@ -129,7 +130,7 @@ class DocumentTest(tests.Test): def test_update(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def prop_1(self, value): @@ -153,7 +154,7 @@ class DocumentTest(tests.Test): def test_delete(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(prefix='P') def prop(self, value): @@ -186,7 +187,7 @@ class DocumentTest(tests.Test): def test_populate(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def prop(self, value): @@ -232,7 +233,7 @@ class DocumentTest(tests.Test): def test_populate_IgnoreBadDocuments(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def prop(self, value): @@ -273,7 +274,7 @@ class DocumentTest(tests.Test): def test_create_with_guid(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def prop(self, value): @@ -293,7 +294,7 @@ class DocumentTest(tests.Test): def test_seqno(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def prop(self, value): @@ -333,7 +334,7 @@ class DocumentTest(tests.Test): def test_diff(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def prop(self, value): @@ -434,9 +435,9 @@ class DocumentTest(tests.Test): def test_diff_IgnoreCalcProps(self): - class Document(document.Document): + class Document(db.Resource): - @db.indexed_property(slot=1, permissions=db.ACCESS_PUBLIC | db.ACCESS_CALC) + @db.indexed_property(slot=1, acl=ACL.PUBLIC | ACL.CALC) def prop(self, value): return value @@ -465,7 +466,7 @@ class DocumentTest(tests.Test): def test_diff_Exclude(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def prop(self, value): @@ -499,7 +500,7 @@ class DocumentTest(tests.Test): URL = 'http://src.sugarlabs.org/robots.txt' URL_content = urllib2.urlopen(URL).read() - class Document(document.Document): + class Document(db.Resource): @db.blob_property() def blob(self, value): @@ -528,7 +529,7 @@ class DocumentTest(tests.Test): def test_diff_Filter(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(prefix='P') def prop(self, value): @@ -555,7 +556,7 @@ class DocumentTest(tests.Test): def test_diff_GroupBy(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(slot=1, prefix='P') def prop(self, value): @@ -584,7 +585,7 @@ class DocumentTest(tests.Test): def test_merge_New(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def prop(self, value): @@ -650,7 +651,7 @@ class DocumentTest(tests.Test): def test_merge_Update(self): - class Document(document.Document): + class Document(db.Resource): @db.blob_property() def blob(self, value): @@ -729,7 +730,7 @@ class DocumentTest(tests.Test): def test_merge_SeqnoLessMode(self): - class Document(document.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def prop(self, value): @@ -788,7 +789,7 @@ class DocumentTest(tests.Test): def test_merge_AvoidCalculatedBlobs(self): - class Document(document.Document): + class Document(db.Resource): @db.blob_property() def blob(self, value): @@ -810,7 +811,7 @@ class DocumentTest(tests.Test): def test_merge_Blobs(self): - class Document(document.Document): + class Document(db.Resource): @db.blob_property() def blob(self, value): @@ -847,7 +848,7 @@ class DocumentTest(tests.Test): def test_wipe(self): - class Document(document.Document): + class Document(db.Resource): pass directory = Directory(tests.tmpdir, Document, IndexWriter) diff --git a/tests/units/db/router.py b/tests/units/db/router.py deleted file mode 100755 index 5147c71..0000000 --- a/tests/units/db/router.py +++ /dev/null @@ -1,533 +0,0 @@ -#!/usr/bin/env python -# sugar-lint: disable - -import re -import os -import time -import json -import urllib2 -import hashlib -import tempfile -from email.utils import formatdate -from cStringIO import StringIO -from os.path import exists - -from __init__ import tests, src_root - -from sugar_network import db, node, static, toolkit -from sugar_network.db.router import Router, _Request, _parse_accept_language, route, _filename -from sugar_network.toolkit import util, default_lang, http -from sugar_network.resources.user import User -from sugar_network.resources.volume import Volume, Resource -from sugar_network import client as local - - -class RouterTest(tests.Test): - - def test_StreamedResponse(self): - - class CommandsProcessor(db.CommandsProcessor): - - @db.volume_command() - def get_stream(self, response): - return StringIO('stream') - - cp = CommandsProcessor() - router = Router(cp) - - response = router({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'GET', - }, - lambda *args: None) - self.assertEqual('stream', ''.join([i for i in response])) - - def test_EmptyResponse(self): - - class CommandsProcessor(db.CommandsProcessor): - - @db.volume_command(cmd='1', mime_type='application/octet-stream') - def get_binary(self, response): - pass - - @db.volume_command(cmd='2', mime_type='application/json') - def get_json(self, response): - pass - - @db.volume_command(cmd='3') - def no_get(self, response): - pass - - cp = CommandsProcessor() - router = Router(cp) - - response = router({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'GET', - 'QUERY_STRING': 'cmd=1', - }, - lambda *args: None) - self.assertEqual('', ''.join([i for i in response])) - - response = router({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'GET', - 'QUERY_STRING': 'cmd=2', - }, - lambda *args: None) - self.assertEqual('null', ''.join([i for i in response])) - - response = router({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'GET', - 'QUERY_STRING': 'cmd=3', - }, - lambda *args: None) - self.assertEqual('', ''.join([i for i in response])) - - def test_StatusWOResult(self): - - class Status(http.Status): - status = '001 Status' - headers = {'status-header': 'value'} - - class CommandsProcessor(db.CommandsProcessor): - - @db.volume_command(method='GET') - def get(self, response): - raise Status('Status-Error') - - router = Router(CommandsProcessor()) - - response = [] - reply = router({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'GET', - }, - lambda status, headers: response.extend([status, dict(headers)])) - error = json.dumps({'request': '/', 'error': 'Status-Error'}) - self.assertEqual(error, ''.join([i for i in reply])) - self.assertEqual([ - '001 Status', - {'content-length': str(len(error)), 'content-type': 'application/json', 'status-header': 'value'}, - ], - response) - - def test_StatusWResult(self): - - class Status(http.Status): - status = '001 Status' - headers = {'status-header': 'value'} - result = 'result' - - class CommandsProcessor(db.CommandsProcessor): - - @db.volume_command(method='GET') - def get(self, response): - raise Status('Status-Error') - - router = Router(CommandsProcessor()) - - response = [] - reply = router({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'GET', - }, - lambda status, headers: response.extend([status, dict(headers)])) - error = 'result' - self.assertEqual(error, ''.join([i for i in reply])) - self.assertEqual([ - '001 Status', - {'content-length': str(len(error)), 'status-header': 'value'}, - ], - response) - - def test_StatusPass(self): - - class StatusPass(http.StatusPass): - status = '001 StatusPass' - headers = {'statuspass-header': 'value'} - result = 'result' - - class CommandsProcessor(db.CommandsProcessor): - - @db.volume_command(method='GET') - def get(self, response): - raise StatusPass('Status-Error') - - router = Router(CommandsProcessor()) - - response = [] - reply = router({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'GET', - }, - lambda status, headers: response.extend([status, dict(headers)])) - error = '' - self.assertEqual(error, ''.join([i for i in reply])) - self.assertEqual([ - '001 StatusPass', - {'content-length': str(len(error)), 'statuspass-header': 'value'}, - ], - response) - - def test_BlobsRedirects(self): - URL = 'http://sugarlabs.org' - - class CommandsProcessor(db.CommandsProcessor): - - @db.volume_command(method='GET') - def get(self, response): - return db.PropertyMetadata(url=URL) - - router = Router(CommandsProcessor()) - - response = [] - reply = router({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'GET', - }, - lambda status, headers: response.extend([status, dict(headers)])) - error = '' - self.assertEqual(error, ''.join([i for i in reply])) - self.assertEqual([ - '303 See Other', - {'content-length': '0', 'location': URL}, - ], - response) - - def test_LastModified(self): - - class CommandsProcessor(db.CommandsProcessor): - - @db.volume_command(method='GET') - def get(self, request): - request.response.last_modified = 10 - return 'ok' - - router = Router(CommandsProcessor()) - - response = [] - reply = router({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'GET', - }, - lambda status, headers: response.extend([status, dict(headers)])) - result = 'ok' - self.assertEqual(result, ''.join([i for i in reply])) - self.assertEqual([ - '200 OK', - {'last-modified': formatdate(10, localtime=False, usegmt=True), 'content-length': str(len(result))}, - ], - response) - - def test_IfModifiedSince(self): - - class CommandsProcessor(db.CommandsProcessor): - - @db.volume_command(method='GET') - def get(self, request): - if not request.if_modified_since or request.if_modified_since >= 10: - return 'ok' - else: - raise http.NotModified() - - router = Router(CommandsProcessor()) - - response = [] - reply = router({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'GET', - }, - lambda status, headers: response.extend([status, dict(headers)])) - result = 'ok' - self.assertEqual(result, ''.join([i for i in reply])) - self.assertEqual([ - '200 OK', - {'content-length': str(len(result))}, - ], - response) - - response = [] - reply = router({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'GET', - 'HTTP_IF_MODIFIED_SINCE': formatdate(11, localtime=False, usegmt=True), - }, - lambda status, headers: response.extend([status, dict(headers)])) - result = 'ok' - self.assertEqual(result, ''.join([i for i in reply])) - self.assertEqual([ - '200 OK', - {'content-length': str(len(result))}, - ], - response) - - response = [] - reply = router({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'GET', - 'HTTP_IF_MODIFIED_SINCE': formatdate(9, localtime=False, usegmt=True), - }, - lambda status, headers: response.extend([status, dict(headers)])) - result = '' - self.assertEqual(result, ''.join([i for i in reply])) - self.assertEqual([ - '304 Not Modified', - {'content-length': str(len(result))}, - ], - response) - - def test_Request_MultipleQueryArguments(self): - request = _Request({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'GET', - 'QUERY_STRING': 'a1=v1&a2=v2&a1=v3&a3=v4&a1=v5&a3=v6', - }) - self.assertEqual( - {'a1': ['v1', 'v3', 'v5'], 'a2': 'v2', 'a3': ['v4', 'v6'], 'method': 'GET'}, - request) - - def test_Register_UrlPath(self): - self.assertEqual( - [], - _Request({'REQUEST_METHOD': 'GET', 'PATH_INFO': ''}).path) - self.assertEqual( - [], - _Request({'REQUEST_METHOD': 'GET', 'PATH_INFO': '/'}).path) - self.assertEqual( - ['foo'], - _Request({'REQUEST_METHOD': 'GET', 'PATH_INFO': 'foo'}).path) - self.assertEqual( - ['foo', 'bar'], - _Request({'REQUEST_METHOD': 'GET', 'PATH_INFO': 'foo/bar'}).path) - self.assertEqual( - ['foo', 'bar'], - _Request({'REQUEST_METHOD': 'GET', 'PATH_INFO': '/foo/bar/'}).path) - self.assertEqual( - ['foo', 'bar'], - _Request({'REQUEST_METHOD': 'GET', 'PATH_INFO': '///foo////bar////'}).path) - - def test_Request_FailOnRelativePaths(self): - self.assertRaises(RuntimeError, _Request, {'REQUEST_METHOD': 'GET', 'PATH_INFO': '..'}) - self.assertRaises(RuntimeError, _Request, {'REQUEST_METHOD': 'GET', 'PATH_INFO': '/..'}) - self.assertRaises(RuntimeError, _Request, {'REQUEST_METHOD': 'GET', 'PATH_INFO': '/../'}) - self.assertRaises(RuntimeError, _Request, {'REQUEST_METHOD': 'GET', 'PATH_INFO': '../bar'}) - self.assertRaises(RuntimeError, _Request, {'REQUEST_METHOD': 'GET', 'PATH_INFO': '/foo/../bar'}) - self.assertRaises(RuntimeError, _Request, {'REQUEST_METHOD': 'GET', 'PATH_INFO': '/foo/..'}) - - def test_parse_accept_language(self): - self.assertEqual( - ['ru', 'en', 'es'], - _parse_accept_language(' ru , en , es')) - self.assertEqual( - ['ru', 'en', 'es'], - _parse_accept_language(' en;q=.4 , ru, es;q=0.1')) - self.assertEqual( - ['ru', 'en', 'es'], - _parse_accept_language('ru;q=1,en;q=1,es;q=0.5')) - self.assertEqual( - ['ru-ru', 'es-br'], - _parse_accept_language('ru-RU,es_BR')) - - def test_StaticFiles(self): - router = Router(db.CommandsProcessor()) - local_path = src_root + '/sugar_network/static/httpdocs/images/missing.png' - - response = [] - reply = router({ - 'PATH_INFO': '/static/images/missing.png', - 'REQUEST_METHOD': 'GET', - }, - lambda status, headers: response.extend([status, dict(headers)])) - result = file(local_path).read() - self.assertEqual(result, ''.join([i for i in reply])) - self.assertEqual([ - '200 OK', - { - 'last-modified': formatdate(os.stat(local_path).st_mtime, localtime=False, usegmt=True), - 'content-length': str(len(result)), - 'content-type': 'image/png', - 'content-disposition': 'attachment; filename="missing.png"', - } - ], - response) - - def test_StaticFilesIfModifiedSince(self): - router = Router(db.CommandsProcessor()) - local_path = src_root + '/sugar_network/static/httpdocs/images/missing.png' - mtime = os.stat(local_path).st_mtime - - response = [] - reply = router({ - 'PATH_INFO': '/static/images/missing.png', - 'REQUEST_METHOD': 'GET', - 'HTTP_IF_MODIFIED_SINCE': formatdate(mtime - 1, localtime=False, usegmt=True), - }, - lambda status, headers: response.extend([status, dict(headers)])) - result = file(local_path).read() - self.assertEqual(result, ''.join([i for i in reply])) - self.assertEqual([ - '200 OK', - { - 'last-modified': formatdate(mtime, localtime=False, usegmt=True), - 'content-length': str(len(result)), - 'content-type': 'image/png', - 'content-disposition': 'attachment; filename="missing.png"', - } - ], - response) - - response = [] - reply = router({ - 'PATH_INFO': '/static/images/missing.png', - 'REQUEST_METHOD': 'GET', - 'HTTP_IF_MODIFIED_SINCE': formatdate(mtime, localtime=False, usegmt=True), - }, - lambda status, headers: response.extend([status, dict(headers)])) - result = '' - self.assertEqual(result, ''.join([i for i in reply])) - self.assertEqual([ - '304 Not Modified', - {'content-length': str(len(result))}, - ], - response) - - response = [] - reply = router({ - 'PATH_INFO': '/static/images/missing.png', - 'REQUEST_METHOD': 'GET', - 'HTTP_IF_MODIFIED_SINCE': formatdate(mtime + 1, localtime=False, usegmt=True), - }, - lambda status, headers: response.extend([status, dict(headers)])) - result = '' - self.assertEqual(result, ''.join([i for i in reply])) - self.assertEqual([ - '304 Not Modified', - {'content-length': str(len(result))}, - ], - response) - - def test_JsonpCallback(self): - - class CommandsProcessor(db.CommandsProcessor): - - @db.volume_command(method='GET') - def get(self, request): - return 'ok' - - router = Router(CommandsProcessor()) - - response = [] - reply = router({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'GET', - 'QUERY_STRING': 'callback=foo', - }, - lambda status, headers: response.extend([status, dict(headers)])) - result = 'foo("ok");' - self.assertEqual(result, ''.join([i for i in reply])) - self.assertEqual([ - '200 OK', - {'content-length': str(len(result))}, - ], - response) - - def test_filename(self): - self.assertEqual('Foo', _filename('foo', None)) - self.assertEqual('Foo-Bar', _filename(['foo', 'bar'], None)) - self.assertEqual('FOO-BaR', _filename([' f o o', ' ba r '], None)) - - self.assertEqual('Foo-3', _filename(['foo', 3], None)) - - self.assertEqual('12-3', _filename(['/1/2/', '/3/'], None)) - - self.assertEqual('Foo.png', _filename('foo', 'image/png')) - self.assertEqual('Foo-Bar.gif', _filename(['foo', 'bar'], 'image/gif')) - self.assertEqual('Fake', _filename('fake', 'foo/bar')) - - self.assertEqual('Eng', _filename({default_lang(): 'eng'}, None)) - self.assertEqual('Eng', _filename([{default_lang(): 'eng'}], None)) - self.assertEqual('Bar-1', _filename([{'lang': 'foo', default_lang(): 'bar'}, 1], None)) - - def test_BlobsDisposition(self): - self.touch(('blob.data', 'value')) - - class CommandsProcessor(db.CommandsProcessor): - - @db.volume_command(method='GET', cmd='1') - def cmd1(self, request): - return db.PropertyMetadata(name='foo', blob='blob.data') - - @db.volume_command(method='GET', cmd='2') - def cmd2(self, request): - return db.PropertyMetadata(filename='foo.bar', blob='blob.data') - - router = Router(CommandsProcessor()) - - response = [] - reply = router({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'GET', - 'QUERY_STRING': 'cmd=1', - }, - lambda status, headers: response.extend([status, dict(headers)])) - result = 'value' - self.assertEqual(result, ''.join([i for i in reply])) - self.assertEqual([ - '200 OK', - { - 'last-modified': formatdate(os.stat('blob.data').st_mtime, localtime=False, usegmt=True), - 'content-length': str(len(result)), - 'content-type': 'application/octet-stream', - 'content-disposition': 'attachment; filename="Foo.obj"', - } - ], - response) - - response = [] - reply = router({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'GET', - 'QUERY_STRING': 'cmd=2', - }, - lambda status, headers: response.extend([status, dict(headers)])) - result = 'value' - self.assertEqual(result, ''.join([i for i in reply])) - self.assertEqual([ - '200 OK', - { - 'last-modified': formatdate(os.stat('blob.data').st_mtime, localtime=False, usegmt=True), - 'content-length': str(len(result)), - 'content-type': 'application/octet-stream', - 'content-disposition': 'attachment; filename="foo.bar"', - } - ], - response) - - def test_DoNotOverrideContentLengthForHEAD(self): - - class CommandsProcessor(db.CommandsProcessor): - - @db.route('HEAD', '/') - def head(self, request, response): - response.content_length = 100 - - router = Router(CommandsProcessor()) - - response = [] - reply = router({ - 'PATH_INFO': '/', - 'REQUEST_METHOD': 'HEAD', - }, - lambda status, headers: response.extend([status, dict(headers)])) - self.assertEqual([], [i for i in reply]) - self.assertEqual([ - '200 OK', - {'content-length': '100'}, - ], - response) - - -if __name__ == '__main__': - tests.main() diff --git a/tests/units/db/routes.py b/tests/units/db/routes.py new file mode 100755 index 0000000..3d2d7f1 --- /dev/null +++ b/tests/units/db/routes.py @@ -0,0 +1,1600 @@ +#!/usr/bin/env python +# sugar-lint: disable + +import os +import sys +import time +import shutil +import hashlib +from cStringIO import StringIO +from email.message import Message +from email.utils import formatdate +from os.path import dirname, join, abspath, exists + +src_root = abspath(dirname(__file__)) + +from __init__ import tests + +from sugar_network import db, toolkit +from sugar_network.db.routes import _typecast_prop_value +from sugar_network.db.metadata import Property +from sugar_network.toolkit.router import Router, Request, Response, fallbackroute, Blob, ACL +from sugar_network.toolkit import coroutine, http + + +class RoutesTest(tests.Test): + + def test_PostDefaults(self): + + class Document(db.Resource): + + @db.stored_property(default='default') + def w_default(self, value): + return value + + @db.stored_property() + def wo_default(self, value): + return value + + @db.indexed_property(slot=1, default='not_stored_default') + def not_stored_default(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [Document], lambda event: None) + + self.assertRaises(RuntimeError, self.call, 'POST', ['document'], content={}) + + guid = self.call('POST', ['document'], content={'wo_default': 'wo_default'}) + self.assertEqual('default', self.call('GET', ['document', guid, 'w_default'])) + self.assertEqual('wo_default', self.call('GET', ['document', guid, 'wo_default'])) + self.assertEqual('not_stored_default', self.call('GET', ['document', guid, 'not_stored_default'])) + + def test_Populate(self): + self.touch( + ('document/1/1/guid', '{"value": "1"}'), + ('document/1/1/ctime', '{"value": 1}'), + ('document/1/1/mtime', '{"value": 1}'), + ('document/1/1/seqno', '{"value": 0}'), + + ('document/2/2/guid', '{"value": "2"}'), + ('document/2/2/ctime', '{"value": 2}'), + ('document/2/2/mtime', '{"value": 2}'), + ('document/2/2/seqno', '{"value": 0}'), + ) + + class Document(db.Resource): + pass + + with db.Volume(tests.tmpdir, [Document], lambda event: None) as volume: + for cls in volume.values(): + for __ in cls.populate(): + pass + self.assertEqual( + sorted(['1', '2']), + sorted([i.guid for i in volume['document'].find()[0]])) + + shutil.rmtree('document/index') + + class Document(db.Resource): + pass + + with db.Volume(tests.tmpdir, [Document], lambda event: None) as volume: + for cls in volume.values(): + for __ in cls.populate(): + pass + self.assertEqual( + sorted(['1', '2']), + sorted([i.guid for i in volume['document'].find()[0]])) + + def test_Commands(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, default='') + def prop(self, value): + return value + + @db.indexed_property(prefix='L', localized=True, default='') + def localized_prop(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + self.volume['testdocument'].create({'guid': 'guid'}) + + self.assertEqual({ + 'total': 1, + 'result': [ + {'guid': 'guid', 'prop': ''}, + ], + }, + self.call('GET', path=['testdocument'], reply=['guid', 'prop'])) + + guid_1 = self.call('POST', path=['testdocument'], content={'prop': 'value_1'}) + assert guid_1 + guid_2 = self.call('POST', path=['testdocument'], content={'prop': 'value_2'}) + assert guid_2 + + self.assertEqual( + sorted([ + {'guid': 'guid', 'prop': ''}, + {'guid': guid_1, 'prop': 'value_1'}, + {'guid': guid_2, 'prop': 'value_2'}, + ]), + sorted(self.call('GET', path=['testdocument'], reply=['guid', 'prop'])['result'])) + + self.call('PUT', path=['testdocument', guid_1], content={'prop': 'value_3'}) + + self.assertEqual( + sorted([ + {'guid': 'guid', 'prop': ''}, + {'guid': guid_1, 'prop': 'value_3'}, + {'guid': guid_2, 'prop': 'value_2'}, + ]), + sorted(self.call('GET', path=['testdocument'], reply=['guid', 'prop'])['result'])) + + self.call('DELETE', path=['testdocument', guid_2]) + + self.assertEqual( + sorted([ + {'guid': 'guid', 'prop': ''}, + {'guid': guid_1, 'prop': 'value_3'}, + ]), + sorted(self.call('GET', path=['testdocument'], reply=['guid', 'prop'])['result'])) + + self.assertRaises(http.NotFound, self.call, 'GET', path=['testdocument', guid_2]) + + self.assertEqual( + {'guid': guid_1, 'prop': 'value_3'}, + self.call('GET', path=['testdocument', guid_1], reply=['guid', 'prop'])) + + self.assertEqual( + 'value_3', + self.call('GET', path=['testdocument', guid_1, 'prop'])) + + def test_SetBLOBs(self): + + class TestDocument(db.Resource): + + @db.blob_property() + def blob(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={}) + + self.call('PUT', path=['testdocument', guid, 'blob'], content='blob1') + self.assertEqual('blob1', file(self.call('GET', path=['testdocument', guid, 'blob'])['blob']).read()) + + self.call('PUT', path=['testdocument', guid, 'blob'], content_stream=StringIO('blob2')) + self.assertEqual('blob2', file(self.call('GET', path=['testdocument', guid, 'blob'])['blob']).read()) + + self.call('PUT', path=['testdocument', guid, 'blob'], content=None) + self.assertRaises(http.NotFound, self.call, 'GET', path=['testdocument', guid, 'blob']) + + def test_SetBLOBsByMeta(self): + + class TestDocument(db.Resource): + + @db.blob_property(mime_type='default') + def blob(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={}) + + self.assertRaises(RuntimeError, self.call, 'PUT', path=['testdocument', guid, 'blob'], + content={}, content_type='application/json') + self.assertRaises(http.NotFound, self.call, 'GET', path=['testdocument', guid, 'blob']) + + self.touch('file') + self.assertRaises(RuntimeError, self.call, 'PUT', path=['testdocument', guid, 'blob'], + content={'blob': 'file'}, content_type='application/json') + self.assertRaises(http.NotFound, self.call, 'GET', path=['testdocument', guid, 'blob']) + + self.call('PUT', path=['testdocument', guid, 'blob'], + content={'url': 'foo', 'bar': 'probe'}, content_type='application/json') + blob = self.call('GET', path=['testdocument', guid, 'blob']) + self.assertEqual('foo', blob['url']) + assert 'bar' not in blob + + def test_RemoveBLOBs(self): + + class TestDocument(db.Resource): + + @db.blob_property(mime_type='default') + def blob(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={'blob': 'blob'}) + + self.assertEqual('blob', file(self.call('GET', path=['testdocument', guid, 'blob'])['blob']).read()) + + self.call('PUT', path=['testdocument', guid, 'blob']) + self.assertRaises(http.NotFound, self.call, 'GET', path=['testdocument', guid, 'blob']) + + def test_RemoveTempBLOBFilesOnFails(self): + + class TestDocument(db.Resource): + + @db.blob_property(mime_type='default') + def blob(self, value): + return value + + @blob.setter + def blob(self, value): + raise RuntimeError() + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={}) + + self.assertRaises(RuntimeError, self.call, 'PUT', path=['testdocument', guid, 'blob'], content='probe') + self.assertEqual(0, len(os.listdir('tmp'))) + + def test_SetBLOBsWithMimeType(self): + + class TestDocument(db.Resource): + + @db.blob_property(mime_type='default') + def blob(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={}) + + self.call('PUT', path=['testdocument', guid, 'blob'], content='blob1') + self.assertEqual('default', self.call('GET', path=['testdocument', guid, 'blob'])['mime_type']) + self.assertEqual('default', self.response.content_type) + + self.call('PUT', path=['testdocument', guid, 'blob'], content='blob1', content_type='foo') + self.assertEqual('foo', self.call('GET', path=['testdocument', guid, 'blob'])['mime_type']) + self.assertEqual('foo', self.response.content_type) + + def test_GetBLOBs(self): + + class TestDocument(db.Resource): + + @db.blob_property() + def blob(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={}) + self.call('PUT', path=['testdocument', guid, 'blob'], content='blob') + + blob_path = tests.tmpdir + '/testdocument/%s/%s/blob' % (guid[:2], guid) + blob_meta = { + 'seqno': 2, + 'blob': blob_path + '.blob', + 'blob_size': 4, + 'digest': hashlib.sha1('blob').hexdigest(), + 'mime_type': 'application/octet-stream', + 'mtime': int(os.stat(blob_path).st_mtime), + } + + self.assertEqual('blob', file(self.call('GET', path=['testdocument', guid, 'blob'])['blob']).read()) + + self.assertEqual( + {'guid': guid, 'blob': 'http://localhost/testdocument/%s/blob' % guid}, + self.call('GET', path=['testdocument', guid], reply=['guid', 'blob'], host='localhost')) + + self.assertEqual([ + {'guid': guid, 'blob': 'http://localhost/testdocument/%s/blob' % guid}, + ], + self.call('GET', path=['testdocument'], reply=['guid', 'blob'], host='localhost')['result']) + + def test_GetBLOBsByUrls(self): + + class TestDocument(db.Resource): + + @db.blob_property() + def blob(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={}) + + self.assertRaises(http.NotFound, self.call, 'GET', path=['testdocument', guid, 'blob']) + self.assertEqual( + {'blob': 'http://127.0.0.1/testdocument/%s/blob' % guid}, + self.call('GET', path=['testdocument', guid], reply=['blob'], host='127.0.0.1')) + self.assertEqual([ + {'blob': 'http://127.0.0.1/testdocument/%s/blob' % guid}, + ], + self.call('GET', path=['testdocument'], reply=['blob'], host='127.0.0.1')['result']) + + self.call('PUT', path=['testdocument', guid, 'blob'], content='file') + self.assertEqual('file', file(self.call('GET', path=['testdocument', guid, 'blob'])['blob']).read()) + self.assertEqual( + {'blob': 'http://127.0.0.1/testdocument/%s/blob' % guid}, + self.call('GET', path=['testdocument', guid], reply=['blob'], host='127.0.0.1')) + self.assertEqual([ + {'blob': 'http://127.0.0.1/testdocument/%s/blob' % guid}, + ], + self.call('GET', path=['testdocument'], reply=['blob'], host='127.0.0.1')['result']) + + self.call('PUT', path=['testdocument', guid, 'blob'], content={'url': 'http://foo'}, + content_type='application/json') + self.assertEqual('http://foo', self.call('GET', path=['testdocument', guid, 'blob'])['url']) + self.assertEqual( + {'blob': 'http://foo'}, + self.call('GET', path=['testdocument', guid], reply=['blob'], host='127.0.0.1')) + self.assertEqual([ + {'blob': 'http://foo'}, + ], + self.call('GET', path=['testdocument'], reply=['blob'], host='127.0.0.1')['result']) + + def test_CommandsGetAbsentBlobs(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, default='') + def prop(self, value): + return value + + @db.blob_property() + def blob(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + + guid = self.call('POST', path=['testdocument'], content={'prop': 'value'}) + self.assertEqual('value', self.call('GET', path=['testdocument', guid, 'prop'])) + self.assertRaises(http.NotFound, self.call, 'GET', path=['testdocument', guid, 'blob']) + self.assertEqual( + {'blob': 'http://localhost/testdocument/%s/blob' % guid}, + self.call('GET', path=['testdocument', guid], reply=['blob'], host='localhost')) + + def test_Command_ReplyForGET(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, default='') + def prop(self, value): + return value + + @db.indexed_property(prefix='L', localized=True, default='') + def localized_prop(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={'prop': 'value'}) + + self.assertEqual( + ['guid', 'prop'], + self.call('GET', path=['testdocument', guid], reply=['guid', 'prop']).keys()) + + self.assertEqual( + ['guid'], + self.call('GET', path=['testdocument'])['result'][0].keys()) + + self.assertEqual( + sorted(['guid', 'prop']), + sorted(self.call('GET', path=['testdocument'], reply=['prop', 'guid'])['result'][0].keys())) + + self.assertEqual( + sorted(['prop']), + sorted(self.call('GET', path=['testdocument'], reply=['prop'])['result'][0].keys())) + + def test_DecodeBeforeSetting(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, typecast=int) + def prop(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + + guid = self.call('POST', path=['testdocument'], content={'prop': '-1'}) + self.assertEqual(-1, self.call('GET', path=['testdocument', guid, 'prop'])) + + def test_LocalizedSet(self): + toolkit._default_lang = 'en' + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, default='') + def prop(self, value): + return value + + @db.indexed_property(prefix='L', localized=True, default='') + def localized_prop(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + directory = self.volume['testdocument'] + + guid = directory.create({'localized_prop': 'value_raw'}) + self.assertEqual({'en': 'value_raw'}, directory.get(guid)['localized_prop']) + self.assertEqual( + [guid], + [i.guid for i in directory.find(0, 100, localized_prop='value_raw')[0]]) + + directory.update(guid, {'localized_prop': 'value_raw2'}) + self.assertEqual({'en': 'value_raw2'}, directory.get(guid)['localized_prop']) + self.assertEqual( + [guid], + [i.guid for i in directory.find(0, 100, localized_prop='value_raw2')[0]]) + + guid = self.call('POST', path=['testdocument'], accept_language=['ru'], content={'localized_prop': 'value_ru'}) + self.assertEqual({'ru': 'value_ru'}, directory.get(guid)['localized_prop']) + self.assertEqual( + [guid], + [i.guid for i in directory.find(0, 100, localized_prop='value_ru')[0]]) + + self.call('PUT', path=['testdocument', guid], accept_language=['en'], content={'localized_prop': 'value_en'}) + self.assertEqual({'ru': 'value_ru', 'en': 'value_en'}, directory.get(guid)['localized_prop']) + self.assertEqual( + [guid], + [i.guid for i in directory.find(0, 100, localized_prop='value_ru')[0]]) + self.assertEqual( + [guid], + [i.guid for i in directory.find(0, 100, localized_prop='value_en')[0]]) + + def test_LocalizedGet(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, default='') + def prop(self, value): + return value + + @db.indexed_property(prefix='L', localized=True, default='') + def localized_prop(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + directory = self.volume['testdocument'] + + guid = self.call('POST', path=['testdocument'], content={ + 'localized_prop': { + 'ru': 'value_ru', + 'es': 'value_es', + 'en': 'value_en', + }, + }) + + toolkit._default_lang = 'en' + + self.assertEqual( + {'localized_prop': 'value_en'}, + self.call('GET', path=['testdocument', guid], reply=['localized_prop'])) + self.assertEqual( + {'localized_prop': 'value_ru'}, + self.call('GET', path=['testdocument', guid], accept_language=['ru'], reply=['localized_prop'])) + self.assertEqual( + 'value_ru', + self.call('GET', path=['testdocument', guid, 'localized_prop'], accept_language=['ru', 'es'])) + self.assertEqual( + [{'localized_prop': 'value_ru'}], + self.call('GET', path=['testdocument'], accept_language=['foo', 'ru', 'es'], reply=['localized_prop'])['result']) + + self.assertEqual( + {'localized_prop': 'value_ru'}, + self.call('GET', path=['testdocument', guid], accept_language=['ru-RU'], reply=['localized_prop'])) + self.assertEqual( + 'value_ru', + self.call('GET', path=['testdocument', guid, 'localized_prop'], accept_language=['ru-RU', 'es'])) + self.assertEqual( + [{'localized_prop': 'value_ru'}], + self.call('GET', path=['testdocument'], accept_language=['foo', 'ru-RU', 'es'], reply=['localized_prop'])['result']) + + self.assertEqual( + {'localized_prop': 'value_es'}, + self.call('GET', path=['testdocument', guid], accept_language=['es'], reply=['localized_prop'])) + self.assertEqual( + 'value_es', + self.call('GET', path=['testdocument', guid, 'localized_prop'], accept_language=['es', 'ru'])) + self.assertEqual( + [{'localized_prop': 'value_es'}], + self.call('GET', path=['testdocument'], accept_language=['foo', 'es', 'ru'], reply=['localized_prop'])['result']) + + self.assertEqual( + {'localized_prop': 'value_en'}, + self.call('GET', path=['testdocument', guid], accept_language=['fr'], reply=['localized_prop'])) + self.assertEqual( + 'value_en', + self.call('GET', path=['testdocument', guid, 'localized_prop'], accept_language=['fr', 'za'])) + self.assertEqual( + [{'localized_prop': 'value_en'}], + self.call('GET', path=['testdocument'], accept_language=['foo', 'fr', 'za'], reply=['localized_prop'])['result']) + + toolkit._default_lang = 'foo' + fallback_lang = sorted(['ru', 'es', 'en'])[0] + + self.assertEqual( + {'localized_prop': 'value_%s' % fallback_lang}, + self.call('GET', path=['testdocument', guid], accept_language=['fr'], reply=['localized_prop'])) + self.assertEqual( + 'value_%s' % fallback_lang, + self.call('GET', path=['testdocument', guid, 'localized_prop'], accept_language=['fr', 'za'])) + self.assertEqual( + [{'localized_prop': 'value_%s' % fallback_lang}], + self.call('GET', path=['testdocument'], accept_language=['foo', 'fr', 'za'], reply=['localized_prop'])['result']) + + def test_OpenByModuleName(self): + self.touch( + ('foo/bar.py', [ + 'from sugar_network import db', + 'class Bar(db.Resource): pass', + ]), + ('foo/__init__.py', ''), + ) + sys.path.insert(0, '.') + + volume = db.Volume('.', ['foo.bar'], lambda event: None) + assert exists('bar/index') + volume['bar'].find() + volume.close() + + def test_Command_GetBlobSetByUrl(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, default='') + def prop(self, value): + return value + + @db.blob_property() + def blob(self, value): + return value + + @db.indexed_property(prefix='L', localized=True, default='') + def localized_prop(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={}) + self.call('PUT', path=['testdocument', guid, 'blob'], url='http://sugarlabs.org') + + self.assertEqual( + 'http://sugarlabs.org', + self.call('GET', path=['testdocument', guid, 'blob'])['url']) + + def test_on_create(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, default='') + def prop(self, value): + return value + + @db.indexed_property(prefix='L', localized=True, default='') + def localized_prop(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + + ts = int(time.time()) + guid = self.call('POST', path=['testdocument'], content={}) + assert self.volume['testdocument'].get(guid)['ctime'] in range(ts - 1, ts + 1) + assert self.volume['testdocument'].get(guid)['mtime'] in range(ts - 1, ts + 1) + + def test_on_create_Override(self): + + class Routes(db.Routes): + + def on_create(self, request, props, event): + props['prop'] = 'overriden' + db.Routes.on_create(self, request, props, event) + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, default='') + def prop(self, value): + return value + + @db.indexed_property(prefix='L', localized=True, default='') + def localized_prop(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + + guid = self.call('POST', ['testdocument'], content={'prop': 'foo'}, routes=Routes) + self.assertEqual('overriden', self.volume['testdocument'].get(guid)['prop']) + + self.call('PUT', ['testdocument', guid], content={'prop': 'bar'}, routes=Routes) + self.assertEqual('bar', self.volume['testdocument'].get(guid)['prop']) + + def test_on_update(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, default='') + def prop(self, value): + return value + + @db.indexed_property(prefix='L', localized=True, default='') + def localized_prop(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={}) + prev_mtime = self.volume['testdocument'].get(guid)['mtime'] + + time.sleep(1) + + self.call('PUT', path=['testdocument', guid], content={'prop': 'probe'}) + assert self.volume['testdocument'].get(guid)['mtime'] - prev_mtime >= 1 + + def test_on_update_Override(self): + + class Routes(db.Routes): + + def on_update(self, request, props, event): + props['prop'] = 'overriden' + db.Routes.on_update(self, request, props, event) + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, default='') + def prop(self, value): + return value + + @db.indexed_property(prefix='L', localized=True, default='') + def localized_prop(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + + guid = self.call('POST', ['testdocument'], content={'prop': 'foo'}, routes=Routes) + self.assertEqual('foo', self.volume['testdocument'].get(guid)['prop']) + + self.call('PUT', ['testdocument', guid], content={'prop': 'bar'}, routes=Routes) + self.assertEqual('overriden', self.volume['testdocument'].get(guid)['prop']) + + def __test_DoNotPassGuidsForCreate(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, default='') + def prop(self, value): + return value + + @db.indexed_property(prefix='L', localized=True, default='') + def localized_prop(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + self.assertRaises(http.Forbidden, self.call, 'POST', path=['testdocument'], content={'guid': 'foo'}) + guid = self.call('POST', path=['testdocument'], content={}) + assert guid + + def test_seqno(self): + + class Document1(db.Resource): + pass + + class Document2(db.Resource): + pass + + volume = db.Volume(tests.tmpdir, [Document1, Document2], lambda event: None) + + assert not exists('seqno') + self.assertEqual(0, volume.seqno.value) + + volume['document1'].create({'guid': '1'}) + self.assertEqual(1, volume['document1'].get('1')['seqno']) + volume['document2'].create({'guid': '1'}) + self.assertEqual(2, volume['document2'].get('1')['seqno']) + volume['document1'].create({'guid': '2'}) + self.assertEqual(3, volume['document1'].get('2')['seqno']) + volume['document2'].create({'guid': '2'}) + self.assertEqual(4, volume['document2'].get('2')['seqno']) + + self.assertEqual(4, volume.seqno.value) + assert not exists('seqno') + volume.seqno.commit() + assert exists('seqno') + volume = db.Volume(tests.tmpdir, [Document1, Document2], lambda event: None) + self.assertEqual(4, volume.seqno.value) + + def test_Events(self): + db.index_flush_threshold.value = 0 + db.index_flush_timeout.value = 0 + + class Document1(db.Resource): + + @db.indexed_property(slot=1, default='') + def prop(self, value): + pass + + class Document2(db.Resource): + + @db.indexed_property(slot=1, default='') + def prop(self, value): + pass + + @db.blob_property() + def blob(self, value): + return value + + self.touch( + ('document1/1/1/guid', '{"value": "1"}'), + ('document1/1/1/ctime', '{"value": 1}'), + ('document1/1/1/mtime', '{"value": 1}'), + ('document1/1/1/prop', '{"value": ""}'), + ('document1/1/1/seqno', '{"value": 0}'), + ) + + events = [] + volume = db.Volume(tests.tmpdir, [Document1, Document2], lambda event: events.append(event)) + coroutine.sleep(.1) + + mtime = int(os.stat('document1/index/mtime').st_mtime) + self.assertEqual([ + {'event': 'commit', 'resource': 'document1', 'mtime': mtime}, + {'event': 'populate', 'resource': 'document1', 'mtime': mtime}, + ], + events) + del events[:] + + volume['document1'].create({'guid': 'guid1'}) + volume['document2'].create({'guid': 'guid2'}) + self.assertEqual([ + {'event': 'create', 'resource': 'document1', 'guid': 'guid1'}, + {'event': 'create', 'resource': 'document2', 'guid': 'guid2'}, + ], + events) + del events[:] + + volume['document1'].update('guid1', {'prop': 'foo'}) + volume['document2'].update('guid2', {'prop': 'bar'}) + self.assertEqual([ + {'event': 'update', 'resource': 'document1', 'guid': 'guid1'}, + {'event': 'update', 'resource': 'document2', 'guid': 'guid2'}, + ], + events) + del events[:] + + volume['document1'].delete('guid1') + self.assertEqual([ + {'event': 'delete', 'resource': 'document1', 'guid': 'guid1'}, + ], + events) + del events[:] + + volume['document1'].commit() + mtime1 = int(os.stat('document1/index/mtime').st_mtime) + volume['document2'].commit() + mtime2 = int(os.stat('document2/index/mtime').st_mtime) + + self.assertEqual([ + {'event': 'commit', 'resource': 'document1', 'mtime': mtime1}, + {'event': 'commit', 'resource': 'document2', 'mtime': mtime2}, + ], + events) + + def test_PermissionsNoWrite(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, default='', acl=ACL.READ) + def prop(self, value): + pass + + @db.blob_property(acl=ACL.READ) + def blob(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={}) + + self.assertRaises(http.Forbidden, self.call, 'POST', path=['testdocument'], content={'prop': 'value'}) + self.assertRaises(http.Forbidden, self.call, 'PUT', path=['testdocument', guid], content={'prop': 'value'}) + self.assertRaises(http.Forbidden, self.call, 'PUT', path=['testdocument', guid], content={'blob': 'value'}) + self.assertRaises(http.Forbidden, self.call, 'PUT', path=['testdocument', guid, 'prop'], content='value') + self.assertRaises(http.Forbidden, self.call, 'PUT', path=['testdocument', guid, 'blob'], content='value') + + def test_BlobsWritePermissions(self): + + class TestDocument(db.Resource): + + @db.blob_property(acl=ACL.CREATE | ACL.WRITE) + def blob1(self, value): + return value + + @db.blob_property(acl=ACL.CREATE) + def blob2(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + + guid = self.call('POST', path=['testdocument'], content={}) + self.call('PUT', path=['testdocument', guid], content={'blob1': 'value1', 'blob2': 'value2'}) + self.call('PUT', path=['testdocument', guid], content={'blob1': 'value1'}) + self.assertRaises(http.Forbidden, self.call, 'PUT', path=['testdocument', guid], content={'blob2': 'value2_'}) + + guid = self.call('POST', path=['testdocument'], content={}) + self.call('PUT', path=['testdocument', guid, 'blob1'], content='value1') + self.call('PUT', path=['testdocument', guid, 'blob2'], content='value2') + self.call('PUT', path=['testdocument', guid, 'blob1'], content='value1_') + self.assertRaises(http.Forbidden, self.call, 'PUT', path=['testdocument', guid, 'blob2'], content='value2_') + + def test_properties_OverrideGet(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, default='1') + def prop1(self, value): + return value + + @db.indexed_property(slot=2, default='2') + def prop2(self, value): + return -1 + + @db.blob_property() + def blob(self, meta): + meta['blob'] = 'new-blob' + return meta + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={}) + self.touch(('new-blob', 'new-blob')) + self.call('PUT', path=['testdocument', guid, 'blob'], content='old-blob') + + self.assertEqual( + 'new-blob', + self.call('GET', path=['testdocument', guid, 'blob'])['blob']) + self.assertEqual( + '1', + self.call('GET', path=['testdocument', guid, 'prop1'])) + self.assertEqual( + -1, + self.call('GET', path=['testdocument', guid, 'prop2'])) + self.assertEqual( + {'prop1': '1', 'prop2': -1}, + self.call('GET', path=['testdocument', guid], reply=['prop1', 'prop2'])) + + def test_properties_OverrideSet(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, default='1') + def prop(self, value): + return value + + @prop.setter + def prop(self, value): + return '_%s' % value + + @db.blob_property() + def blob1(self, meta): + return meta + + @blob1.setter + def blob1(self, value): + return Blob({'url': file(value['blob']).read()}) + + @db.blob_property() + def blob2(self, meta): + return meta + + @blob2.setter + def blob2(self, value): + with toolkit.NamedTemporaryFile(delete=False) as f: + f.write(' %s ' % file(value['blob']).read()) + value['blob'] = f.name + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={}) + + self.assertEqual('_1', self.call('GET', path=['testdocument', guid, 'prop'])) + self.assertRaises(http.NotFound, self.call, 'GET', path=['testdocument', guid, 'blob1']) + + self.call('PUT', path=['testdocument', guid, 'prop'], content='2') + self.assertEqual('_2', self.call('GET', path=['testdocument', guid, 'prop'])) + self.assertRaises(http.NotFound, self.call, 'GET', path=['testdocument', guid, 'blob1']) + + self.call('PUT', path=['testdocument', guid], content={'prop': 3}) + self.assertEqual('_3', self.call('GET', path=['testdocument', guid, 'prop'])) + self.assertRaises(http.NotFound, self.call, 'GET', path=['testdocument', guid, 'blob1']) + + self.call('PUT', path=['testdocument', guid, 'blob1'], content='blob_url') + self.assertEqual('blob_url', self.call('GET', path=['testdocument', guid, 'blob1'])['url']) + + guid = self.call('POST', path=['testdocument'], content={'blob2': 'foo'}) + self.assertEqual(' foo ', file(self.call('GET', path=['testdocument', guid, 'blob2'])['blob']).read()) + + self.call('PUT', path=['testdocument', guid, 'blob2'], content='bar') + self.assertEqual(' bar ', file(self.call('GET', path=['testdocument', guid, 'blob2'])['blob']).read()) + + def test_properties_CallSettersAtTheEnd(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, typecast=int) + def prop1(self, value): + return value + + @prop1.setter + def prop1(self, value): + return self['prop3'] + value + + @db.indexed_property(slot=2, typecast=int) + def prop2(self, value): + return value + + @prop2.setter + def prop2(self, value): + return self['prop3'] - value + + @db.indexed_property(slot=3, typecast=int) + def prop3(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={'prop1': 1, 'prop2': 2, 'prop3': 3}) + self.assertEqual(4, self.call('GET', path=['testdocument', guid, 'prop1'])) + self.assertEqual(1, self.call('GET', path=['testdocument', guid, 'prop2'])) + + def test_properties_PopulateRequiredPropsInSetters(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, typecast=int) + def prop1(self, value): + return value + + @prop1.setter + def prop1(self, value): + self['prop2'] = value + 1 + return value + + @db.indexed_property(slot=2, typecast=int) + def prop2(self, value): + return value + + @db.blob_property() + def prop3(self, value): + return value + + @prop3.setter + def prop3(self, value): + self['prop1'] = -1 + self['prop2'] = -2 + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={'prop1': 1}) + self.assertEqual(1, self.call('GET', path=['testdocument', guid, 'prop1'])) + self.assertEqual(2, self.call('GET', path=['testdocument', guid, 'prop2'])) + + def test_properties_PopulateRequiredPropsInBlobSetter(self): + + class TestDocument(db.Resource): + + @db.blob_property() + def blob(self, value): + return value + + @blob.setter + def blob(self, value): + self['prop1'] = 1 + self['prop2'] = 2 + return value + + @db.indexed_property(slot=1, typecast=int) + def prop1(self, value): + return value + + @db.indexed_property(slot=2, typecast=int) + def prop2(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={'blob': ''}) + self.assertEqual(1, self.call('GET', path=['testdocument', guid, 'prop1'])) + self.assertEqual(2, self.call('GET', path=['testdocument', guid, 'prop2'])) + + def __test_SubCall(self): + + class TestDocument(db.Resource): + + @db.blob_property(mime_type='application/json') + def blob(self, value): + return value + + @blob.setter + def blob(self, value): + blob = file(value['blob']).read() + if '!' not in blob: + meta = self.meta('blob') + if meta: + blob = file(meta['blob']).read() + blob + with toolkit.NamedTemporaryFile(delete=False) as f: + f.write(blob) + value['blob'] = f.name + coroutine.spawn(self.post, blob) + return value + + def post(self, value): + self.request.call('PUT', path=['testdocument', self.guid, 'blob'], content=value + '!') + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + + guid = self.call('POST', path=['testdocument'], content={'blob': '0'}) + coroutine.dispatch() + self.assertEqual('0!', file(self.call('GET', path=['testdocument', guid, 'blob'])['blob']).read()) + + self.call('PUT', path=['testdocument', guid, 'blob'], content='1') + coroutine.dispatch() + self.assertEqual('0!1!', file(self.call('GET', path=['testdocument', guid, 'blob'])['blob']).read()) + + def test_Group(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1) + def prop(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + + self.call('POST', path=['testdocument'], content={'prop': 1}) + self.call('POST', path=['testdocument'], content={'prop': 2}) + self.call('POST', path=['testdocument'], content={'prop': 1}) + + self.assertEqual( + sorted([{'prop': 1}, {'prop': 2}]), + sorted(self.call('GET', path=['testdocument'], reply='prop', group_by='prop')['result'])) + + def test_CallSetterEvenIfThereIsNoCreatePermissions(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, acl=ACL.READ, default=0) + def prop(self, value): + return value + + @prop.setter + def prop(self, value): + return value + 1 + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + + self.assertRaises(http.Forbidden, self.call, 'POST', path=['testdocument'], content={'prop': 1}) + + guid = self.call('POST', path=['testdocument'], content={}) + self.assertEqual(1, self.call('GET', path=['testdocument', guid, 'prop'])) + + def test_ReturnDefualtsForMissedProps(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, default='default') + def prop(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', path=['testdocument'], content={'prop': 'set'}) + + self.assertEqual( + [{'prop': 'set'}], + self.call('GET', path=['testdocument'], reply='prop')['result']) + self.assertEqual( + {'prop': 'set'}, + self.call('GET', path=['testdocument', guid], reply='prop')) + self.assertEqual( + 'set', + self.call('GET', path=['testdocument', guid, 'prop'])) + + os.unlink('testdocument/%s/%s/prop' % (guid[:2], guid)) + + self.assertEqual( + [{'prop': 'default'}], + self.call('GET', path=['testdocument'], reply='prop')['result']) + self.assertEqual( + {'prop': 'default'}, + self.call('GET', path=['testdocument', guid], reply='prop')) + self.assertEqual( + 'default', + self.call('GET', path=['testdocument', guid, 'prop'])) + + def test_PopulateNonDefualtPropsInSetters(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1) + def prop1(self, value): + return value + + @db.indexed_property(slot=2, default='default') + def prop2(self, value): + return all + + @prop2.setter + def prop2(self, value): + if value != 'default': + self['prop1'] = value + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + + self.assertRaises(RuntimeError, self.call, 'POST', path=['testdocument'], content={}) + + guid = self.call('POST', path=['testdocument'], content={'prop2': 'value2'}) + self.assertEqual('value2', self.call('GET', path=['testdocument', guid, 'prop1'])) + + def test_prop_meta(self): + + class TestDocument(db.Resource): + + @db.indexed_property(slot=1, default='') + def prop(self, value): + return value + + @db.blob_property() + def blob1(self, value): + return value + + @db.blob_property() + def blob2(self, value): + return value + + @blob2.setter + def blob2(self, value): + return {'url': 'http://new', 'foo': 'bar', 'blob_size': 100} + + self.volume = db.Volume(tests.tmpdir, [TestDocument], lambda event: None) + guid = self.call('POST', ['testdocument'], content = {'prop': 'prop', 'blob1': 'blob', 'blob2': ''}) + + assert self.call('HEAD', ['testdocument', guid, 'prop']) is None + meta = self.volume['testdocument'].get(guid).meta('prop') + meta.pop('value') + self.assertEqual(meta, self.response.meta) + self.assertEqual(formatdate(meta['mtime'], localtime=False, usegmt=True), self.response.last_modified) + + assert self.call('HEAD', ['testdocument', guid, 'blob1'], host='localhost') is None + meta = self.volume['testdocument'].get(guid).meta('blob1') + meta.pop('blob') + meta['url'] = 'http://localhost/testdocument/%s/blob1' % guid + self.assertEqual(meta, self.response.meta) + self.assertEqual(len('blob'), self.response.content_length) + self.assertEqual(formatdate(meta['mtime'], localtime=False, usegmt=True), self.response.last_modified) + + assert self.call('HEAD', ['testdocument', guid, 'blob2']) is None + meta = self.volume['testdocument'].get(guid).meta('blob2') + self.assertEqual(meta, self.response.meta) + self.assertEqual(100, self.response.content_length) + self.assertEqual(formatdate(meta['mtime'], localtime=False, usegmt=True), self.response.last_modified) + + def test_DefaultAuthor(self): + + class User(db.Resource): + + @db.indexed_property(slot=1) + def name(self, value): + return value + + class Document(db.Resource): + pass + + self.volume = db.Volume('db', [User, Document]) + + guid = self.call('POST', ['document'], content={}, principal='user') + self.assertEqual( + [{'name': 'user', 'role': 2}], + self.call('GET', ['document', guid, 'author'])) + self.assertEqual( + {'user': {'role': 2, 'order': 0}}, + self.volume['document'].get(guid)['author']) + + self.volume['user'].create({'guid': 'user', 'color': '', 'pubkey': '', 'name': 'User'}) + + guid = self.call('POST', ['document'], content={}, principal='user') + self.assertEqual( + [{'guid': 'user', 'name': 'User', 'role': 3}], + self.call('GET', ['document', guid, 'author'])) + self.assertEqual( + {'user': {'name': 'User', 'role': 3, 'order': 0}}, + self.volume['document'].get(guid)['author']) + + def test_FindByAuthor(self): + + class User(db.Resource): + + @db.indexed_property(slot=1) + def name(self, value): + return value + + class Document(db.Resource): + pass + + self.volume = db.Volume('db', [User, Document]) + + self.volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'UserName1'}) + self.volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User Name2'}) + self.volume['user'].create({'guid': 'user3', 'color': '', 'pubkey': '', 'name': 'User Name 3'}) + + guid1 = self.call('POST', ['document'], content={}, principal='user1') + guid2 = self.call('POST', ['document'], content={}, principal='user2') + guid3 = self.call('POST', ['document'], content={}, principal='user3') + + self.assertEqual(sorted([ + {'guid': guid1}, + ]), + self.call('GET', ['document'], author='UserName1')['result']) + + self.assertEqual(sorted([ + {'guid': guid1}, + ]), + sorted(self.call('GET', ['document'], query='author:UserName')['result'])) + self.assertEqual(sorted([ + {'guid': guid1}, + {'guid': guid2}, + {'guid': guid3}, + ]), + sorted(self.call('GET', ['document'], query='author:User')['result'])) + self.assertEqual(sorted([ + {'guid': guid2}, + {'guid': guid3}, + ]), + sorted(self.call('GET', ['document'], query='author:Name')['result'])) + + def test_PreserveAuthorsOrder(self): + + class User(db.Resource): + + @db.indexed_property(slot=1) + def name(self, value): + return value + + class Document(db.Resource): + pass + + self.volume = db.Volume('db', [User, Document]) + + self.volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'}) + self.volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User2'}) + self.volume['user'].create({'guid': 'user3', 'color': '', 'pubkey': '', 'name': 'User3'}) + + guid = self.call('POST', ['document'], content={}, principal='user1') + self.call('PUT', ['document', guid], cmd='useradd', user='user2', role=0) + self.call('PUT', ['document', guid], cmd='useradd', user='user3', role=0) + + self.assertEqual([ + {'guid': 'user1', 'name': 'User1', 'role': 3}, + {'guid': 'user2', 'name': 'User2', 'role': 1}, + {'guid': 'user3', 'name': 'User3', 'role': 1}, + ], + self.call('GET', ['document', guid, 'author'])) + self.assertEqual({ + 'user1': {'name': 'User1', 'role': 3, 'order': 0}, + 'user2': {'name': 'User2', 'role': 1, 'order': 1}, + 'user3': {'name': 'User3', 'role': 1, 'order': 2}, + }, + self.volume['document'].get(guid)['author']) + + self.call('PUT', ['document', guid], cmd='userdel', user='user2', principal='user1') + self.call('PUT', ['document', guid], cmd='useradd', user='user2', role=0) + + self.assertEqual([ + {'guid': 'user1', 'name': 'User1', 'role': 3}, + {'guid': 'user3', 'name': 'User3', 'role': 1}, + {'guid': 'user2', 'name': 'User2', 'role': 1}, + ], + self.call('GET', ['document', guid, 'author'])) + self.assertEqual({ + 'user1': {'name': 'User1', 'role': 3, 'order': 0}, + 'user3': {'name': 'User3', 'role': 1, 'order': 2}, + 'user2': {'name': 'User2', 'role': 1, 'order': 3}, + }, + self.volume['document'].get(guid)['author']) + + self.call('PUT', ['document', guid], cmd='userdel', user='user2', principal='user1') + self.call('PUT', ['document', guid], cmd='useradd', user='user2', role=0) + + self.assertEqual([ + {'guid': 'user1', 'name': 'User1', 'role': 3}, + {'guid': 'user3', 'name': 'User3', 'role': 1}, + {'guid': 'user2', 'name': 'User2', 'role': 1}, + ], + self.call('GET', ['document', guid, 'author'])) + self.assertEqual({ + 'user1': {'name': 'User1', 'role': 3, 'order': 0}, + 'user3': {'name': 'User3', 'role': 1, 'order': 2}, + 'user2': {'name': 'User2', 'role': 1, 'order': 3}, + }, + self.volume['document'].get(guid)['author']) + + self.call('PUT', ['document', guid], cmd='userdel', user='user3', principal='user1') + self.call('PUT', ['document', guid], cmd='useradd', user='user3', role=0) + + self.assertEqual([ + {'guid': 'user1', 'name': 'User1', 'role': 3}, + {'guid': 'user2', 'name': 'User2', 'role': 1}, + {'guid': 'user3', 'name': 'User3', 'role': 1}, + ], + self.call('GET', ['document', guid, 'author'])) + self.assertEqual({ + 'user1': {'name': 'User1', 'role': 3, 'order': 0}, + 'user2': {'name': 'User2', 'role': 1, 'order': 3}, + 'user3': {'name': 'User3', 'role': 1, 'order': 4}, + }, + self.volume['document'].get(guid)['author']) + + def test_AddUser(self): + + class User(db.Resource): + + @db.indexed_property(slot=1) + def name(self, value): + return value + + class Document(db.Resource): + pass + + self.volume = db.Volume('db', [User, Document]) + + self.volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'}) + self.volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User2'}) + + guid = self.call('POST', ['document'], content={}, principal='user1') + self.assertEqual([ + {'guid': 'user1', 'name': 'User1', 'role': 3}, + ], + self.call('GET', ['document', guid, 'author'])) + self.assertEqual({ + 'user1': {'name': 'User1', 'role': 3, 'order': 0}, + }, + self.volume['document'].get(guid)['author']) + + self.call('PUT', ['document', guid], cmd='useradd', user='user2', role=2) + self.assertEqual([ + {'guid': 'user1', 'name': 'User1', 'role': 3}, + {'guid': 'user2', 'name': 'User2', 'role': 3}, + ], + self.call('GET', ['document', guid, 'author'])) + self.assertEqual({ + 'user1': {'name': 'User1', 'role': 3, 'order': 0}, + 'user2': {'name': 'User2', 'role': 3, 'order': 1}, + }, + self.volume['document'].get(guid)['author']) + + self.call('PUT', ['document', guid], cmd='useradd', user='User3', role=3) + self.assertEqual([ + {'guid': 'user1', 'name': 'User1', 'role': 3}, + {'guid': 'user2', 'name': 'User2', 'role': 3}, + {'name': 'User3', 'role': 2}, + ], + self.call('GET', ['document', guid, 'author'])) + self.assertEqual({ + 'user1': {'name': 'User1', 'role': 3, 'order': 0}, + 'user2': {'name': 'User2', 'role': 3, 'order': 1}, + 'User3': {'role': 2, 'order': 2}, + }, + self.volume['document'].get(guid)['author']) + + self.call('PUT', ['document', guid], cmd='useradd', user='User4', role=4) + self.assertEqual([ + {'guid': 'user1', 'name': 'User1', 'role': 3}, + {'guid': 'user2', 'name': 'User2', 'role': 3}, + {'name': 'User3', 'role': 2}, + {'name': 'User4', 'role': 0}, + ], + self.call('GET', ['document', guid, 'author'])) + self.assertEqual({ + 'user1': {'name': 'User1', 'role': 3, 'order': 0}, + 'user2': {'name': 'User2', 'role': 3, 'order': 1}, + 'User3': {'role': 2, 'order': 2}, + 'User4': {'role': 0, 'order': 3}, + }, + self.volume['document'].get(guid)['author']) + + def test_UpdateAuthor(self): + + class User(db.Resource): + + @db.indexed_property(slot=1) + def name(self, value): + return value + + class Document(db.Resource): + pass + + self.volume = db.Volume('db', [User, Document]) + + self.volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'}) + guid = self.call('POST', ['document'], content={}, principal='user1') + + self.call('PUT', ['document', guid], cmd='useradd', user='User2', role=0) + self.assertEqual([ + {'guid': 'user1', 'name': 'User1', 'role': 3}, + {'name': 'User2', 'role': 0}, + ], + self.call('GET', ['document', guid, 'author'])) + self.assertEqual({ + 'user1': {'name': 'User1', 'role': 3, 'order': 0}, + 'User2': {'role': 0, 'order': 1}, + }, + self.volume['document'].get(guid)['author']) + + self.call('PUT', ['document', guid], cmd='useradd', user='user1', role=0) + self.assertEqual([ + {'guid': 'user1', 'name': 'User1', 'role': 1}, + {'name': 'User2', 'role': 0}, + ], + self.call('GET', ['document', guid, 'author'])) + self.assertEqual({ + 'user1': {'name': 'User1', 'role': 1, 'order': 0}, + 'User2': {'role': 0, 'order': 1}, + }, + self.volume['document'].get(guid)['author']) + + self.call('PUT', ['document', guid], cmd='useradd', user='User2', role=2) + self.assertEqual([ + {'guid': 'user1', 'name': 'User1', 'role': 1}, + {'name': 'User2', 'role': 2}, + ], + self.call('GET', ['document', guid, 'author'])) + self.assertEqual({ + 'user1': {'name': 'User1', 'role': 1, 'order': 0}, + 'User2': {'role': 2, 'order': 1}, + }, + self.volume['document'].get(guid)['author']) + + def test_DelUser(self): + + class User(db.Resource): + + @db.indexed_property(slot=1) + def name(self, value): + return value + + class Document(db.Resource): + pass + + self.volume = db.Volume('db', [User, Document]) + + self.volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'}) + self.volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User2'}) + guid = self.call('POST', ['document'], content={}, principal='user1') + self.call('PUT', ['document', guid], cmd='useradd', user='user2') + self.call('PUT', ['document', guid], cmd='useradd', user='User3') + self.assertEqual([ + {'guid': 'user1', 'name': 'User1', 'role': 3}, + {'guid': 'user2', 'name': 'User2', 'role': 1}, + {'name': 'User3', 'role': 0}, + ], + self.call('GET', ['document', guid, 'author'])) + self.assertEqual({ + 'user1': {'name': 'User1', 'role': 3, 'order': 0}, + 'user2': {'name': 'User2', 'role': 1, 'order': 1}, + 'User3': {'role': 0, 'order': 2}, + }, + self.volume['document'].get(guid)['author']) + + # Do not remove yourself + self.assertRaises(RuntimeError, self.call, 'PUT', ['document', guid], cmd='userdel', user='user1', principal='user1') + self.assertRaises(RuntimeError, self.call, 'PUT', ['document', guid], cmd='userdel', user='user2', principal='user2') + + self.call('PUT', ['document', guid], cmd='userdel', user='user1', principal='user2') + self.assertEqual([ + {'guid': 'user2', 'name': 'User2', 'role': 1}, + {'name': 'User3', 'role': 0}, + ], + self.call('GET', ['document', guid, 'author'])) + self.assertEqual({ + 'user2': {'name': 'User2', 'role': 1, 'order': 1}, + 'User3': {'role': 0, 'order': 2}, + }, + self.volume['document'].get(guid)['author']) + + self.call('PUT', ['document', guid], cmd='userdel', user='User3', principal='user2') + self.assertEqual([ + {'guid': 'user2', 'name': 'User2', 'role': 1}, + ], + self.call('GET', ['document', guid, 'author'])) + self.assertEqual({ + 'user2': {'name': 'User2', 'role': 1, 'order': 1}, + }, + self.volume['document'].get(guid)['author']) + + def test_typecast_prop_value(self): + prop = Property('prop', typecast=int) + self.assertEqual(1, _typecast_prop_value(prop.typecast, 1)) + self.assertEqual(1, _typecast_prop_value(prop.typecast, 1.1)) + self.assertEqual(1, _typecast_prop_value(prop.typecast, '1')) + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, '1.0') + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, '') + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, None) + + prop = Property('prop', typecast=float) + self.assertEqual(1.0, _typecast_prop_value(prop.typecast, 1)) + self.assertEqual(1.1, _typecast_prop_value(prop.typecast, 1.1)) + self.assertEqual(1.0, _typecast_prop_value(prop.typecast, '1')) + self.assertEqual(1.1, _typecast_prop_value(prop.typecast, '1.1')) + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, '') + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, None) + + prop = Property('prop', typecast=bool) + self.assertEqual(False, _typecast_prop_value(prop.typecast, 0)) + self.assertEqual(True, _typecast_prop_value(prop.typecast, 1)) + self.assertEqual(True, _typecast_prop_value(prop.typecast, 1.1)) + self.assertEqual(True, _typecast_prop_value(prop.typecast, '1')) + self.assertEqual(True, _typecast_prop_value(prop.typecast, 'false')) + self.assertEqual(False, _typecast_prop_value(prop.typecast, '')) + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, None) + + prop = Property('prop', typecast=[int]) + self.assertEqual((1,), _typecast_prop_value(prop.typecast, 1)) + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, None) + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, '') + self.assertEqual((), _typecast_prop_value(prop.typecast, [])) + self.assertEqual((123,), _typecast_prop_value(prop.typecast, '123')) + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, 'a') + self.assertEqual((123, 4, 5), _typecast_prop_value(prop.typecast, ['123', 4, 5.6])) + + prop = Property('prop', typecast=[1, 2]) + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, 0) + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, None) + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, '') + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, 'A') + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, '3') + self.assertEqual(1, _typecast_prop_value(prop.typecast, 1)) + self.assertEqual(2, _typecast_prop_value(prop.typecast, 2)) + self.assertEqual(1, _typecast_prop_value(prop.typecast, '1')) + + prop = Property('prop', typecast=[str]) + self.assertEqual(('',), _typecast_prop_value(prop.typecast, '')) + self.assertEqual(('',), _typecast_prop_value(prop.typecast, [''])) + self.assertEqual((), _typecast_prop_value(prop.typecast, [])) + + prop = Property('prop', typecast=[]) + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, None) + self.assertEqual(('',), _typecast_prop_value(prop.typecast, '')) + self.assertEqual(('',), _typecast_prop_value(prop.typecast, [''])) + self.assertEqual((), _typecast_prop_value(prop.typecast, [])) + self.assertEqual(('0',), _typecast_prop_value(prop.typecast, 0)) + self.assertEqual(('',), _typecast_prop_value(prop.typecast, '')) + self.assertEqual(('foo',), _typecast_prop_value(prop.typecast, 'foo')) + + prop = Property('prop', typecast=[['A', 'B', 'C']]) + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, '') + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, ['']) + self.assertEqual((), _typecast_prop_value(prop.typecast, [])) + self.assertEqual(('A', 'B', 'C'), _typecast_prop_value(prop.typecast, ['A', 'B', 'C'])) + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, ['a']) + self.assertRaises(ValueError, _typecast_prop_value, prop.typecast, ['A', 'x']) + + prop = Property('prop', typecast=bool) + self.assertEqual(True, _typecast_prop_value(prop.typecast, True)) + self.assertEqual(False, _typecast_prop_value(prop.typecast, False)) + self.assertEqual(True, _typecast_prop_value(prop.typecast, 'False')) + self.assertEqual(True, _typecast_prop_value(prop.typecast, '0')) + + prop = Property('prop', typecast=[['A', 'B', 'C']]) + self.assertEqual(('A', 'B', 'C'), _typecast_prop_value(prop.typecast, ['A', 'B', 'C'])) + + prop = Property('prop', typecast=lambda x: x + 1) + self.assertEqual(1, _typecast_prop_value(prop.typecast, 0)) + + def call(self, method=None, path=None, + accept_language=None, content=None, content_stream=None, cmd=None, + content_type=None, host=None, request=None, routes=db.Routes, principal=None, + **kwargs): + if request is None: + request = Request({ + 'REQUEST_METHOD': method, + 'PATH_INFO': '/'.join([''] + path), + 'HTTP_ACCEPT_LANGUAGE': ','.join(accept_language or []), + 'HTTP_HOST': host, + 'wsgi.input': content_stream, + }) + request.cmd = cmd + request.content = content + request.content_type = content_type + if request.content_stream is not None: + request.content_length = len(request.content_stream.getvalue()) + request.update(kwargs) + request.principal = principal + router = Router(routes(self.volume)) + self.response = Response() + return router._call(request, self.response) + + +if __name__ == '__main__': + tests.main() diff --git a/tests/units/db/storage.py b/tests/units/db/storage.py index 149ed03..6eb62e5 100755 --- a/tests/units/db/storage.py +++ b/tests/units/db/storage.py @@ -11,11 +11,10 @@ from os.path import exists from __init__ import tests -from sugar_network.db import env from sugar_network.db.metadata import Metadata, StoredProperty from sugar_network.db.metadata import BlobProperty from sugar_network.db.storage import Storage -from sugar_network.toolkit import BUFFER_SIZE, util +from sugar_network.toolkit import BUFFER_SIZE class StorageTest(tests.Test): diff --git a/tests/units/db/volume.py b/tests/units/db/volume.py deleted file mode 100755 index b968069..0000000 --- a/tests/units/db/volume.py +++ /dev/null @@ -1,1223 +0,0 @@ -#!/usr/bin/env python -# sugar-lint: disable - -import os -import sys -import time -import shutil -import hashlib -from cStringIO import StringIO -from email.message import Message -from email.utils import formatdate -from os.path import dirname, join, abspath, exists - -src_root = abspath(dirname(__file__)) - -from __init__ import tests - -from sugar_network import db, toolkit -from sugar_network.db import env -from sugar_network.db.volume import VolumeCommands -from sugar_network.toolkit import coroutine, http, util - - -class VolumeTest(tests.Test): - - def setUp(self): - tests.Test.setUp(self) - self.response = db.Response() - - def test_PostDefaults(self): - - class Document(db.Document): - - @db.stored_property(default='default') - def w_default(self, value): - return value - - @db.stored_property() - def wo_default(self, value): - return value - - @db.indexed_property(slot=1, default='not_stored_default') - def not_stored_default(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [Document]) - - self.assertRaises(RuntimeError, self.call, 'POST', document='document', content={}) - - guid = self.call('POST', document='document', content={'wo_default': 'wo_default'}) - self.assertEqual('default', self.call('GET', document='document', guid=guid, prop='w_default')) - self.assertEqual('wo_default', self.call('GET', document='document', guid=guid, prop='wo_default')) - self.assertEqual('not_stored_default', self.call('GET', document='document', guid=guid, prop='not_stored_default')) - - def test_Populate(self): - self.touch( - ('document/1/1/guid', '{"value": "1"}'), - ('document/1/1/ctime', '{"value": 1}'), - ('document/1/1/mtime', '{"value": 1}'), - ('document/1/1/seqno', '{"value": 0}'), - - ('document/2/2/guid', '{"value": "2"}'), - ('document/2/2/ctime', '{"value": 2}'), - ('document/2/2/mtime', '{"value": 2}'), - ('document/2/2/seqno', '{"value": 0}'), - ) - - class Document(db.Document): - pass - - with db.Volume(tests.tmpdir, [Document]) as volume: - for cls in volume.values(): - for __ in cls.populate(): - pass - self.assertEqual( - sorted(['1', '2']), - sorted([i.guid for i in volume['document'].find()[0]])) - - shutil.rmtree('document/index') - - class Document(db.Document): - pass - - with db.Volume(tests.tmpdir, [Document]) as volume: - for cls in volume.values(): - for __ in cls.populate(): - pass - self.assertEqual( - sorted(['1', '2']), - sorted([i.guid for i in volume['document'].find()[0]])) - - def test_Commands(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, default='') - def prop(self, value): - return value - - @db.indexed_property(prefix='L', localized=True, default='') - def localized_prop(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - self.volume['testdocument'].create({'guid': 'guid'}) - - self.assertEqual({ - 'total': 1, - 'result': [ - {'guid': 'guid', 'prop': ''}, - ], - }, - self.call('GET', document='testdocument', reply=['guid', 'prop'])) - - guid_1 = self.call('POST', document='testdocument', content={'prop': 'value_1'}) - assert guid_1 - guid_2 = self.call('POST', document='testdocument', content={'prop': 'value_2'}) - assert guid_2 - - self.assertEqual( - sorted([ - {'guid': 'guid', 'prop': ''}, - {'guid': guid_1, 'prop': 'value_1'}, - {'guid': guid_2, 'prop': 'value_2'}, - ]), - sorted(self.call('GET', document='testdocument', reply=['guid', 'prop'])['result'])) - - self.call('PUT', document='testdocument', guid=guid_1, content={'prop': 'value_3'}) - - self.assertEqual( - sorted([ - {'guid': 'guid', 'prop': ''}, - {'guid': guid_1, 'prop': 'value_3'}, - {'guid': guid_2, 'prop': 'value_2'}, - ]), - sorted(self.call('GET', document='testdocument', reply=['guid', 'prop'])['result'])) - - self.call('DELETE', document='testdocument', guid=guid_2) - - self.assertEqual( - sorted([ - {'guid': 'guid', 'prop': ''}, - {'guid': guid_1, 'prop': 'value_3'}, - ]), - sorted(self.call('GET', document='testdocument', reply=['guid', 'prop'])['result'])) - - self.assertRaises(http.NotFound, self.call, 'GET', document='testdocument', guid=guid_2) - - self.assertEqual( - {'guid': guid_1, 'prop': 'value_3'}, - self.call('GET', document='testdocument', guid=guid_1, reply=['guid', 'prop'])) - - self.assertEqual( - 'value_3', - self.call('GET', document='testdocument', guid=guid_1, prop='prop')) - - def test_SetBLOBs(self): - - class TestDocument(db.Document): - - @db.blob_property() - def blob(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call('POST', document='testdocument', content={}) - - self.call('PUT', document='testdocument', guid=guid, prop='blob', content='blob1') - self.assertEqual('blob1', file(self.call('GET', document='testdocument', guid=guid, prop='blob')['blob']).read()) - - self.call('PUT', document='testdocument', guid=guid, prop='blob', content_stream=StringIO('blob2')) - self.assertEqual('blob2', file(self.call('GET', document='testdocument', guid=guid, prop='blob')['blob']).read()) - - self.call('PUT', document='testdocument', guid=guid, prop='blob', content=None) - self.assertRaises(http.NotFound, self.call, 'GET', document='testdocument', guid=guid, prop='blob') - - def test_SetBLOBsByMeta(self): - - class TestDocument(db.Document): - - @db.blob_property(mime_type='default') - def blob(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call('POST', document='testdocument', content={}) - - self.assertRaises(RuntimeError, self.call, 'PUT', document='testdocument', guid=guid, prop='blob', - content={}, content_type='application/json') - self.assertRaises(http.NotFound, self.call, 'GET', document='testdocument', guid=guid, prop='blob') - - self.touch('file') - self.assertRaises(RuntimeError, self.call, 'PUT', document='testdocument', guid=guid, prop='blob', - content={'blob': 'file'}, content_type='application/json') - self.assertRaises(http.NotFound, self.call, 'GET', document='testdocument', guid=guid, prop='blob') - - self.call('PUT', document='testdocument', guid=guid, prop='blob', - content={'url': 'foo', 'bar': 'probe'}, content_type='application/json') - blob = self.call('GET', document='testdocument', guid=guid, prop='blob') - self.assertEqual('foo', blob['url']) - assert 'bar' not in blob - - def test_RemoveBLOBs(self): - - class TestDocument(db.Document): - - @db.blob_property(mime_type='default') - def blob(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call('POST', document='testdocument', content={'blob': 'blob'}) - - self.assertEqual('blob', file(self.call('GET', document='testdocument', guid=guid, prop='blob')['blob']).read()) - - self.call('PUT', document='testdocument', guid=guid, prop='blob') - self.assertRaises(http.NotFound, self.call, 'GET', document='testdocument', guid=guid, prop='blob') - - def test_RemoveTempBLOBFilesOnFails(self): - - class TestDocument(db.Document): - - @db.blob_property(mime_type='default') - def blob(self, value): - return value - - @blob.setter - def blob(self, value): - raise RuntimeError() - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call('POST', document='testdocument', content={}) - - self.assertRaises(RuntimeError, self.call, 'PUT', document='testdocument', guid=guid, prop='blob', content='probe') - self.assertEqual(0, len(os.listdir('tmp'))) - - def test_SetBLOBsWithMimeType(self): - - class TestDocument(db.Document): - - @db.blob_property(mime_type='default') - def blob(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call('POST', document='testdocument', content={}) - - self.call('PUT', document='testdocument', guid=guid, prop='blob', content='blob1') - self.assertEqual('default', self.call('GET', document='testdocument', guid=guid, prop='blob')['mime_type']) - self.assertEqual('default', self.response.content_type) - - self.call('PUT', document='testdocument', guid=guid, prop='blob', content='blob1', content_type='foo') - self.assertEqual('foo', self.call('GET', document='testdocument', guid=guid, prop='blob')['mime_type']) - self.assertEqual('foo', self.response.content_type) - - def test_GetBLOBs(self): - - class TestDocument(db.Document): - - @db.blob_property() - def blob(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call('POST', document='testdocument', content={}) - self.call('PUT', document='testdocument', guid=guid, prop='blob', content='blob') - - blob_path = tests.tmpdir + '/testdocument/%s/%s/blob' % (guid[:2], guid) - blob_meta = { - 'seqno': 2, - 'blob': blob_path + '.blob', - 'blob_size': 4, - 'digest': hashlib.sha1('blob').hexdigest(), - 'mime_type': 'application/octet-stream', - 'mtime': int(os.stat(blob_path).st_mtime), - } - - self.assertEqual('blob', file(self.call('GET', document='testdocument', guid=guid, prop='blob')['blob']).read()) - - self.assertEqual( - {'guid': guid, 'blob': blob_meta}, - self.call('GET', document='testdocument', guid=guid, reply=['guid', 'blob'])) - - self.assertEqual([ - {'guid': guid, 'blob': blob_meta}, - ], - self.call('GET', document='testdocument', reply=['guid', 'blob'])['result']) - - def test_GetBLOBsByUrls(self): - - class TestDocument(db.Document): - - @db.blob_property() - def blob(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call('POST', document='testdocument', content={}) - - self.assertRaises(http.NotFound, self.call, 'GET', document='testdocument', guid=guid, prop='blob') - self.assertEqual( - {'blob': 'http://127.0.0.1/testdocument/%s/blob' % guid}, - self.call('GET', document='testdocument', guid=guid, reply=['blob'], static_prefix='http://127.0.0.1')) - self.assertEqual([ - {'blob': 'http://127.0.0.1/testdocument/%s/blob' % guid}, - ], - self.call('GET', document='testdocument', reply=['blob'], static_prefix='http://127.0.0.1')['result']) - - self.call('PUT', document='testdocument', guid=guid, prop='blob', content='file') - self.assertEqual('file', file(self.call('GET', document='testdocument', guid=guid, prop='blob')['blob']).read()) - self.assertEqual( - {'blob': 'http://127.0.0.1/testdocument/%s/blob' % guid}, - self.call('GET', document='testdocument', guid=guid, reply=['blob'], static_prefix='http://127.0.0.1')) - self.assertEqual([ - {'blob': 'http://127.0.0.1/testdocument/%s/blob' % guid}, - ], - self.call('GET', document='testdocument', reply=['blob'], static_prefix='http://127.0.0.1')['result']) - - self.call('PUT', document='testdocument', guid=guid, prop='blob', content={'url': 'http://foo'}, - content_type='application/json') - self.assertEqual('http://foo', self.call('GET', document='testdocument', guid=guid, prop='blob')['url']) - self.assertEqual( - {'blob': 'http://foo'}, - self.call('GET', document='testdocument', guid=guid, reply=['blob'], static_prefix='http://127.0.0.1')) - self.assertEqual([ - {'blob': 'http://foo'}, - ], - self.call('GET', document='testdocument', reply=['blob'], static_prefix='http://127.0.0.1')['result']) - - def test_CommandsGetAbsentBlobs(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, default='') - def prop(self, value): - return value - - @db.blob_property() - def blob(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - - guid = self.call('POST', document='testdocument', content={'prop': 'value'}) - self.assertEqual('value', self.call('GET', document='testdocument', guid=guid, prop='prop')) - self.assertRaises(http.NotFound, self.call, 'GET', document='testdocument', guid=guid, prop='blob') - self.assertEqual( - {'blob': db.PropertyMetadata()}, - self.call('GET', document='testdocument', guid=guid, reply=['blob'])) - - def test_Command_ReplyForGET(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, default='') - def prop(self, value): - return value - - @db.indexed_property(prefix='L', localized=True, default='') - def localized_prop(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call('POST', document='testdocument', content={'prop': 'value'}) - - self.assertEqual( - ['guid', 'prop'], - self.call('GET', document='testdocument', guid=guid, reply=['guid', 'prop']).keys()) - - self.assertEqual( - ['guid'], - self.call('GET', document='testdocument')['result'][0].keys()) - - self.assertEqual( - sorted(['guid', 'prop']), - sorted(self.call('GET', document='testdocument', reply=['prop', 'guid'])['result'][0].keys())) - - self.assertEqual( - sorted(['prop']), - sorted(self.call('GET', document='testdocument', reply=['prop'])['result'][0].keys())) - - def test_DecodeBeforeSetting(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, typecast=int) - def prop(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - - guid = self.call(method='POST', document='testdocument', content={'prop': '-1'}) - self.assertEqual(-1, self.call(method='GET', document='testdocument', guid=guid, prop='prop')) - - def test_LocalizedSet(self): - toolkit._default_lang = 'en' - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, default='') - def prop(self, value): - return value - - @db.indexed_property(prefix='L', localized=True, default='') - def localized_prop(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - directory = self.volume['testdocument'] - - guid = directory.create({'localized_prop': 'value_raw'}) - self.assertEqual({'en': 'value_raw'}, directory.get(guid)['localized_prop']) - self.assertEqual( - [guid], - [i.guid for i in directory.find(0, 100, localized_prop='value_raw')[0]]) - - directory.update(guid, {'localized_prop': 'value_raw2'}) - self.assertEqual({'en': 'value_raw2'}, directory.get(guid)['localized_prop']) - self.assertEqual( - [guid], - [i.guid for i in directory.find(0, 100, localized_prop='value_raw2')[0]]) - - guid = self.call('POST', document='testdocument', accept_language=['ru'], content={'localized_prop': 'value_ru'}) - self.assertEqual({'ru': 'value_ru'}, directory.get(guid)['localized_prop']) - self.assertEqual( - [guid], - [i.guid for i in directory.find(0, 100, localized_prop='value_ru')[0]]) - - self.call('PUT', document='testdocument', guid=guid, accept_language=['en'], content={'localized_prop': 'value_en'}) - self.assertEqual({'ru': 'value_ru', 'en': 'value_en'}, directory.get(guid)['localized_prop']) - self.assertEqual( - [guid], - [i.guid for i in directory.find(0, 100, localized_prop='value_ru')[0]]) - self.assertEqual( - [guid], - [i.guid for i in directory.find(0, 100, localized_prop='value_en')[0]]) - - def test_LocalizedGet(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, default='') - def prop(self, value): - return value - - @db.indexed_property(prefix='L', localized=True, default='') - def localized_prop(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - directory = self.volume['testdocument'] - - guid = self.call('POST', document='testdocument', content={ - 'localized_prop': { - 'ru': 'value_ru', - 'es': 'value_es', - 'en': 'value_en', - }, - }) - - toolkit._default_lang = 'en' - - self.assertEqual( - {'localized_prop': 'value_en'}, - self.call('GET', document='testdocument', guid=guid, reply=['localized_prop'])) - self.assertEqual( - {'localized_prop': 'value_ru'}, - self.call('GET', document='testdocument', guid=guid, accept_language=['ru'], reply=['localized_prop'])) - self.assertEqual( - 'value_ru', - self.call('GET', document='testdocument', guid=guid, accept_language=['ru', 'es'], prop='localized_prop')) - self.assertEqual( - [{'localized_prop': 'value_ru'}], - self.call('GET', document='testdocument', accept_language=['foo', 'ru', 'es'], reply=['localized_prop'])['result']) - - self.assertEqual( - {'localized_prop': 'value_ru'}, - self.call('GET', document='testdocument', guid=guid, accept_language=['ru-RU'], reply=['localized_prop'])) - self.assertEqual( - 'value_ru', - self.call('GET', document='testdocument', guid=guid, accept_language=['ru-RU', 'es'], prop='localized_prop')) - self.assertEqual( - [{'localized_prop': 'value_ru'}], - self.call('GET', document='testdocument', accept_language=['foo', 'ru-RU', 'es'], reply=['localized_prop'])['result']) - - self.assertEqual( - {'localized_prop': 'value_es'}, - self.call('GET', document='testdocument', guid=guid, accept_language=['es'], reply=['localized_prop'])) - self.assertEqual( - 'value_es', - self.call('GET', document='testdocument', guid=guid, accept_language=['es', 'ru'], prop='localized_prop')) - self.assertEqual( - [{'localized_prop': 'value_es'}], - self.call('GET', document='testdocument', accept_language=['foo', 'es', 'ru'], reply=['localized_prop'])['result']) - - self.assertEqual( - {'localized_prop': 'value_en'}, - self.call('GET', document='testdocument', guid=guid, accept_language=['fr'], reply=['localized_prop'])) - self.assertEqual( - 'value_en', - self.call('GET', document='testdocument', guid=guid, accept_language=['fr', 'za'], prop='localized_prop')) - self.assertEqual( - [{'localized_prop': 'value_en'}], - self.call('GET', document='testdocument', accept_language=['foo', 'fr', 'za'], reply=['localized_prop'])['result']) - - toolkit._default_lang = 'foo' - fallback_lang = sorted(['ru', 'es', 'en'])[0] - - self.assertEqual( - {'localized_prop': 'value_%s' % fallback_lang}, - self.call('GET', document='testdocument', guid=guid, accept_language=['fr'], reply=['localized_prop'])) - self.assertEqual( - 'value_%s' % fallback_lang, - self.call('GET', document='testdocument', guid=guid, accept_language=['fr', 'za'], prop='localized_prop')) - self.assertEqual( - [{'localized_prop': 'value_%s' % fallback_lang}], - self.call('GET', document='testdocument', accept_language=['foo', 'fr', 'za'], reply=['localized_prop'])['result']) - - def test_OpenByModuleName(self): - self.touch( - ('foo/bar.py', [ - 'from sugar_network import db', - 'class Bar(db.Document): pass', - ]), - ('foo/__init__.py', ''), - ) - sys.path.insert(0, '.') - - volume = db.Volume('.', ['foo.bar']) - assert exists('bar/index') - volume['bar'].find() - volume.close() - - def test_Command_GetBlobSetByUrl(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, default='') - def prop(self, value): - return value - - @db.blob_property() - def blob(self, value): - return value - - @db.indexed_property(prefix='L', localized=True, default='') - def localized_prop(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call('POST', document='testdocument', content={}) - self.call('PUT', document='testdocument', guid=guid, prop='blob', url='http://sugarlabs.org') - - self.assertEqual( - 'http://sugarlabs.org', - self.call('GET', document='testdocument', guid=guid, prop='blob')['url']) - - def test_on_create(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, default='') - def prop(self, value): - return value - - @db.indexed_property(prefix='L', localized=True, default='') - def localized_prop(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - - ts = int(time.time()) - guid = self.call(method='POST', document='testdocument', content={}) - assert self.volume['testdocument'].get(guid)['ctime'] in range(ts - 1, ts + 1) - assert self.volume['testdocument'].get(guid)['mtime'] in range(ts - 1, ts + 1) - - def test_on_create_Override(self): - - class Commands(VolumeCommands): - - def on_create(self, request, props, event): - props['prop'] = 'overriden' - VolumeCommands.on_create(self, request, props, event) - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, default='') - def prop(self, value): - return value - - @db.indexed_property(prefix='L', localized=True, default='') - def localized_prop(self, value): - return value - - volume = db.Volume(tests.tmpdir, [TestDocument]) - cp = Commands(volume) - - request = db.Request(method='POST', document='testdocument') - request.content = {'prop': 'foo'} - guid = cp.call(request, db.Response()) - self.assertEqual('overriden', volume['testdocument'].get(guid)['prop']) - - request = db.Request(method='PUT', document='testdocument', guid=guid) - request.content = {'prop': 'bar'} - cp.call(request, db.Response()) - self.assertEqual('bar', volume['testdocument'].get(guid)['prop']) - - def test_on_update(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, default='') - def prop(self, value): - return value - - @db.indexed_property(prefix='L', localized=True, default='') - def localized_prop(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call(method='POST', document='testdocument', content={}) - prev_mtime = self.volume['testdocument'].get(guid)['mtime'] - - time.sleep(1) - - self.call(method='PUT', document='testdocument', guid=guid, content={'prop': 'probe'}) - assert self.volume['testdocument'].get(guid)['mtime'] - prev_mtime >= 1 - - def test_on_update_Override(self): - - class Commands(VolumeCommands): - - def on_update(self, request, props, event): - props['prop'] = 'overriden' - VolumeCommands.on_update(self, request, props, event) - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, default='') - def prop(self, value): - return value - - @db.indexed_property(prefix='L', localized=True, default='') - def localized_prop(self, value): - return value - - volume = db.Volume(tests.tmpdir, [TestDocument]) - cp = Commands(volume) - - request = db.Request(method='POST', document='testdocument') - request.content = {'prop': 'foo'} - guid = cp.call(request, db.Response()) - self.assertEqual('foo', volume['testdocument'].get(guid)['prop']) - - request = db.Request(method='PUT', document='testdocument', guid=guid) - request.content = {'prop': 'bar'} - cp.call(request, db.Response()) - self.assertEqual('overriden', volume['testdocument'].get(guid)['prop']) - - def __test_DoNotPassGuidsForCreate(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, default='') - def prop(self, value): - return value - - @db.indexed_property(prefix='L', localized=True, default='') - def localized_prop(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - self.assertRaises(http.Forbidden, self.call, method='POST', document='testdocument', content={'guid': 'foo'}) - guid = self.call(method='POST', document='testdocument', content={}) - assert guid - - def test_seqno(self): - - class Document1(db.Document): - pass - - class Document2(db.Document): - pass - - volume = db.Volume(tests.tmpdir, [Document1, Document2]) - - assert not exists('seqno') - self.assertEqual(0, volume.seqno.value) - - volume['document1'].create({'guid': '1'}) - self.assertEqual(1, volume['document1'].get('1')['seqno']) - volume['document2'].create({'guid': '1'}) - self.assertEqual(2, volume['document2'].get('1')['seqno']) - volume['document1'].create({'guid': '2'}) - self.assertEqual(3, volume['document1'].get('2')['seqno']) - volume['document2'].create({'guid': '2'}) - self.assertEqual(4, volume['document2'].get('2')['seqno']) - - self.assertEqual(4, volume.seqno.value) - assert not exists('seqno') - volume.seqno.commit() - assert exists('seqno') - volume = db.Volume(tests.tmpdir, [Document1, Document2]) - self.assertEqual(4, volume.seqno.value) - - def test_Events(self): - env.index_flush_threshold.value = 0 - env.index_flush_timeout.value = 0 - - class Document1(db.Document): - - @db.indexed_property(slot=1, default='') - def prop(self, value): - pass - - class Document2(db.Document): - - @db.indexed_property(slot=1, default='') - def prop(self, value): - pass - - @db.blob_property() - def blob(self, value): - return value - - self.touch( - ('document1/1/1/guid', '{"value": "1"}'), - ('document1/1/1/ctime', '{"value": 1}'), - ('document1/1/1/mtime', '{"value": 1}'), - ('document1/1/1/prop', '{"value": ""}'), - ('document1/1/1/seqno', '{"value": 0}'), - ) - - events = [] - volume = db.Volume(tests.tmpdir, [Document1, Document2]) - volume.connect(lambda event: events.append(event)) - - volume.populate() - mtime = int(os.stat('document1/index/mtime').st_mtime) - self.assertEqual([ - {'event': 'commit', 'document': 'document1', 'mtime': mtime}, - {'event': 'populate', 'document': 'document1', 'mtime': mtime}, - ], - events) - del events[:] - - volume['document1'].create({'guid': 'guid1'}) - volume['document2'].create({'guid': 'guid2'}) - self.assertEqual([ - {'event': 'create', 'document': 'document1', 'guid': 'guid1'}, - {'event': 'create', 'document': 'document2', 'guid': 'guid2'}, - ], - events) - del events[:] - - volume['document1'].update('guid1', {'prop': 'foo'}) - volume['document2'].update('guid2', {'prop': 'bar'}) - self.assertEqual([ - {'event': 'update', 'document': 'document1', 'guid': 'guid1'}, - {'event': 'update', 'document': 'document2', 'guid': 'guid2'}, - ], - events) - del events[:] - - volume['document1'].delete('guid1') - self.assertEqual([ - {'event': 'delete', 'document': 'document1', 'guid': 'guid1'}, - ], - events) - del events[:] - - volume['document1'].commit() - mtime1 = int(os.stat('document1/index/mtime').st_mtime) - volume['document2'].commit() - mtime2 = int(os.stat('document2/index/mtime').st_mtime) - - self.assertEqual([ - {'event': 'commit', 'document': 'document1', 'mtime': mtime1}, - {'event': 'commit', 'document': 'document2', 'mtime': mtime2}, - ], - events) - - def test_PermissionsNoWrite(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, default='', permissions=db.ACCESS_READ) - def prop(self, value): - pass - - @db.blob_property(permissions=db.ACCESS_READ) - def blob(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call('POST', document='testdocument', content={}) - - self.assertRaises(http.Forbidden, self.call, 'POST', document='testdocument', content={'prop': 'value'}) - self.assertRaises(http.Forbidden, self.call, 'PUT', document='testdocument', guid=guid, content={'prop': 'value'}) - self.assertRaises(http.Forbidden, self.call, 'PUT', document='testdocument', guid=guid, content={'blob': 'value'}) - self.assertRaises(http.Forbidden, self.call, 'PUT', document='testdocument', guid=guid, prop='prop', content='value') - self.assertRaises(http.Forbidden, self.call, 'PUT', document='testdocument', guid=guid, prop='blob', content='value') - - def test_BlobsWritePermissions(self): - - class TestDocument(db.Document): - - @db.blob_property(permissions=db.ACCESS_CREATE | db.ACCESS_WRITE) - def blob1(self, value): - return value - - @db.blob_property(permissions=db.ACCESS_CREATE) - def blob2(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - - guid = self.call('POST', document='testdocument', content={}) - self.call('PUT', document='testdocument', guid=guid, content={'blob1': 'value1', 'blob2': 'value2'}) - self.call('PUT', document='testdocument', guid=guid, content={'blob1': 'value1'}) - self.assertRaises(http.Forbidden, self.call, 'PUT', document='testdocument', guid=guid, content={'blob2': 'value2_'}) - - guid = self.call('POST', document='testdocument', content={}) - self.call('PUT', document='testdocument', guid=guid, prop='blob1', content='value1') - self.call('PUT', document='testdocument', guid=guid, prop='blob2', content='value2') - self.call('PUT', document='testdocument', guid=guid, prop='blob1', content='value1_') - self.assertRaises(http.Forbidden, self.call, 'PUT', document='testdocument', guid=guid, prop='blob2', content='value2_') - - def test_properties_OverrideGet(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, default='1') - def prop1(self, value): - return value - - @db.indexed_property(slot=2, default='2') - def prop2(self, value): - return -1 - - @db.blob_property() - def blob(self, meta): - meta['blob'] = 'new-blob' - return meta - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call('POST', document='testdocument', content={}) - self.touch(('new-blob', 'new-blob')) - self.call('PUT', document='testdocument', guid=guid, prop='blob', content='old-blob') - - self.assertEqual( - 'new-blob', - self.call('GET', document='testdocument', guid=guid, prop='blob')['blob']) - self.assertEqual( - '1', - self.call('GET', document='testdocument', guid=guid, prop='prop1')) - self.assertEqual( - -1, - self.call('GET', document='testdocument', guid=guid, prop='prop2')) - self.assertEqual( - {'prop1': '1', 'prop2': -1}, - self.call('GET', document='testdocument', guid=guid, reply=['prop1', 'prop2'])) - - def test_properties_OverrideSet(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, default='1') - def prop(self, value): - return value - - @prop.setter - def prop(self, value): - return '_%s' % value - - @db.blob_property() - def blob1(self, meta): - return meta - - @blob1.setter - def blob1(self, value): - return db.PropertyMetadata(url=file(value['blob']).read()) - - @db.blob_property() - def blob2(self, meta): - return meta - - @blob2.setter - def blob2(self, value): - with util.NamedTemporaryFile(delete=False) as f: - f.write(' %s ' % file(value['blob']).read()) - value['blob'] = f.name - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call('POST', document='testdocument', content={}) - - self.assertEqual('_1', self.call('GET', document='testdocument', guid=guid, prop='prop')) - self.assertRaises(http.NotFound, self.call, 'GET', document='testdocument', guid=guid, prop='blob1') - - self.call('PUT', document='testdocument', guid=guid, prop='prop', content='2') - self.assertEqual('_2', self.call('GET', document='testdocument', guid=guid, prop='prop')) - self.assertRaises(http.NotFound, self.call, 'GET', document='testdocument', guid=guid, prop='blob1') - - self.call('PUT', document='testdocument', guid=guid, content={'prop': 3}) - self.assertEqual('_3', self.call('GET', document='testdocument', guid=guid, prop='prop')) - self.assertRaises(http.NotFound, self.call, 'GET', document='testdocument', guid=guid, prop='blob1') - - self.call('PUT', document='testdocument', guid=guid, prop='blob1', content='blob_url') - self.assertEqual('blob_url', self.call('GET', document='testdocument', guid=guid, prop='blob1')['url']) - - guid = self.call('POST', document='testdocument', content={'blob2': 'foo'}) - self.assertEqual(' foo ', file(self.call('GET', document='testdocument', guid=guid, prop='blob2')['blob']).read()) - - self.call('PUT', document='testdocument', guid=guid, prop='blob2', content='bar') - self.assertEqual(' bar ', file(self.call('GET', document='testdocument', guid=guid, prop='blob2')['blob']).read()) - - def test_properties_CallSettersAtTheEnd(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, typecast=int) - def prop1(self, value): - return value - - @prop1.setter - def prop1(self, value): - return self['prop3'] + value - - @db.indexed_property(slot=2, typecast=int) - def prop2(self, value): - return value - - @prop2.setter - def prop2(self, value): - return self['prop3'] - value - - @db.indexed_property(slot=3, typecast=int) - def prop3(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call('POST', document='testdocument', content={'prop1': 1, 'prop2': 2, 'prop3': 3}) - self.assertEqual(4, self.call('GET', document='testdocument', guid=guid, prop='prop1')) - self.assertEqual(1, self.call('GET', document='testdocument', guid=guid, prop='prop2')) - - def test_properties_PopulateRequiredPropsInSetters(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, typecast=int) - def prop1(self, value): - return value - - @prop1.setter - def prop1(self, value): - self['prop2'] = value + 1 - return value - - @db.indexed_property(slot=2, typecast=int) - def prop2(self, value): - return value - - @db.blob_property() - def prop3(self, value): - return value - - @prop3.setter - def prop3(self, value): - self['prop1'] = -1 - self['prop2'] = -2 - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call('POST', document='testdocument', content={'prop1': 1}) - self.assertEqual(1, self.call('GET', document='testdocument', guid=guid, prop='prop1')) - self.assertEqual(2, self.call('GET', document='testdocument', guid=guid, prop='prop2')) - - def test_properties_PopulateRequiredPropsInBlobSetter(self): - - class TestDocument(db.Document): - - @db.blob_property() - def blob(self, value): - return value - - @blob.setter - def blob(self, value): - self['prop1'] = 1 - self['prop2'] = 2 - return value - - @db.indexed_property(slot=1, typecast=int) - def prop1(self, value): - return value - - @db.indexed_property(slot=2, typecast=int) - def prop2(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call('POST', document='testdocument', content={'blob': ''}) - self.assertEqual(1, self.call('GET', document='testdocument', guid=guid, prop='prop1')) - self.assertEqual(2, self.call('GET', document='testdocument', guid=guid, prop='prop2')) - - def test_SubCall(self): - - class TestDocument(db.Document): - - @db.blob_property(mime_type='application/json') - def blob(self, value): - return value - - @blob.setter - def blob(self, value): - blob = file(value['blob']).read() - if '!' not in blob: - meta = self.meta('blob') - if meta: - blob = file(meta['blob']).read() + blob - with util.NamedTemporaryFile(delete=False) as f: - f.write(blob) - value['blob'] = f.name - coroutine.spawn(self.post, blob) - return value - - def post(self, value): - self.request.call('PUT', document='testdocument', guid=self.guid, prop='blob', content=value + '!') - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - - guid = self.call('POST', document='testdocument', content={'blob': '0'}) - coroutine.dispatch() - self.assertEqual('0!', file(self.call('GET', document='testdocument', guid=guid, prop='blob')['blob']).read()) - - self.call('PUT', document='testdocument', guid=guid, prop='blob', content='1') - coroutine.dispatch() - self.assertEqual('0!1!', file(self.call('GET', document='testdocument', guid=guid, prop='blob')['blob']).read()) - - def test_Group(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1) - def prop(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - - self.call('POST', document='testdocument', content={'prop': 1}) - self.call('POST', document='testdocument', content={'prop': 2}) - self.call('POST', document='testdocument', content={'prop': 1}) - - self.assertEqual( - sorted([{'prop': 1}, {'prop': 2}]), - sorted(self.call('GET', document='testdocument', reply='prop', group_by='prop')['result'])) - - def test_CallSetterEvenIfThereIsNoCreatePermissions(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, permissions=db.ACCESS_READ, default=0) - def prop(self, value): - return value - - @prop.setter - def prop(self, value): - return value + 1 - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - - self.assertRaises(http.Forbidden, self.call, 'POST', document='testdocument', content={'prop': 1}) - - guid = self.call('POST', document='testdocument', content={}) - self.assertEqual(1, self.call('GET', document='testdocument', guid=guid, prop='prop')) - - def test_ReturnDefualtsForMissedProps(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, default='default') - def prop(self, value): - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - guid = self.call('POST', document='testdocument', content={'prop': 'set'}) - - self.assertEqual( - [{'prop': 'set'}], - self.call('GET', document='testdocument', reply='prop')['result']) - self.assertEqual( - {'prop': 'set'}, - self.call('GET', document='testdocument', guid=guid, reply='prop')) - self.assertEqual( - 'set', - self.call('GET', document='testdocument', guid=guid, prop='prop')) - - os.unlink('testdocument/%s/%s/prop' % (guid[:2], guid)) - - self.assertEqual( - [{'prop': 'default'}], - self.call('GET', document='testdocument', reply='prop')['result']) - self.assertEqual( - {'prop': 'default'}, - self.call('GET', document='testdocument', guid=guid, reply='prop')) - self.assertEqual( - 'default', - self.call('GET', document='testdocument', guid=guid, prop='prop')) - - def test_PopulateNonDefualtPropsInSetters(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1) - def prop1(self, value): - return value - - @db.indexed_property(slot=2, default='default') - def prop2(self, value): - return all - - @prop2.setter - def prop2(self, value): - if value != 'default': - self['prop1'] = value - return value - - self.volume = db.Volume(tests.tmpdir, [TestDocument]) - - self.assertRaises(RuntimeError, self.call, 'POST', document='testdocument', content={}) - - guid = self.call('POST', document='testdocument', content={'prop2': 'value2'}) - self.assertEqual('value2', self.call('GET', document='testdocument', guid=guid, prop='prop1')) - - def test_prop_meta(self): - - class TestDocument(db.Document): - - @db.indexed_property(slot=1, default='') - def prop(self, value): - return value - - @db.blob_property() - def blob1(self, value): - return value - - @db.blob_property() - def blob2(self, value): - return value - - @blob2.setter - def blob2(self, value): - return {'url': 'http://new', 'foo': 'bar', 'blob_size': 100} - - volume = db.Volume(tests.tmpdir, [TestDocument]) - cp = VolumeCommands(volume) - - request = db.Request(method='POST', document='testdocument') - request.content = {'prop': 'prop', 'blob1': 'blob', 'blob2': ''} - guid = cp.call(request, db.Response()) - - request = db.Request(method='HEAD', document='testdocument', guid=guid, prop='prop') - response = db.Response() - assert cp.call(request, response) is None - meta = volume['testdocument'].get(guid).meta('prop') - meta.pop('value') - self.assertEqual(meta, response.meta) - self.assertEqual(formatdate(meta['mtime'], localtime=False, usegmt=True), response.last_modified) - - request = db.Request(method='HEAD', document='testdocument', guid=guid, prop='blob1') - request.static_prefix = 'http://localhost' - request.path = ['path'] - response = db.Response() - assert cp.call(request, response) is None - meta = volume['testdocument'].get(guid).meta('blob1') - meta.pop('blob') - meta['url'] = 'http://localhost/path' - self.assertEqual(meta, response.meta) - self.assertEqual(len('blob'), response.content_length) - self.assertEqual(formatdate(meta['mtime'], localtime=False, usegmt=True), response.last_modified) - - request = db.Request(method='HEAD', document='testdocument', guid=guid, prop='blob2') - response = db.Response() - assert cp.call(request, response) is None - meta = volume['testdocument'].get(guid).meta('blob2') - self.assertEqual(meta, response.meta) - self.assertEqual(100, response.content_length) - self.assertEqual(formatdate(meta['mtime'], localtime=False, usegmt=True), response.last_modified) - - def call(self, method, document=None, guid=None, prop=None, - accept_language=None, content=None, content_stream=None, - content_type=None, if_modified_since=None, static_prefix=None, - **kwargs): - - class TestRequest(db.Request): - - content_stream = None - content_length = 0 - - request = TestRequest(**kwargs) - request.static_prefix = static_prefix - request.content = content - request.content_stream = content_stream - request.content_type = content_type - request.accept_language = accept_language - request.if_modified_since = if_modified_since - request['method'] = method - if document: - request['document'] = document - if guid: - request['guid'] = guid - if prop: - request['prop'] = prop - if request.content_stream is not None: - request.content_length = len(request.content_stream.getvalue()) - - self.response = db.Response() - cp = VolumeCommands(self.volume) - return cp.call(request, self.response) - - -if __name__ == '__main__': - tests.main() diff --git a/tests/units/resources/__init__.py b/tests/units/model/__init__.py index 345c327..345c327 100644 --- a/tests/units/resources/__init__.py +++ b/tests/units/model/__init__.py diff --git a/tests/units/resources/__main__.py b/tests/units/model/__main__.py index 4444f07..6d8e563 100644 --- a/tests/units/resources/__main__.py +++ b/tests/units/model/__main__.py @@ -7,7 +7,7 @@ from context import * from implementation import * from review import * from solution import * -from volume import * +from routes import * if __name__ == '__main__': tests.main() diff --git a/tests/units/resources/comment.py b/tests/units/model/comment.py index 9c89517..7cf23b9 100755 --- a/tests/units/resources/comment.py +++ b/tests/units/model/comment.py @@ -4,13 +4,13 @@ from __init__ import tests from sugar_network.client import Client -from sugar_network.resources.user import User -from sugar_network.resources.context import Context -from sugar_network.resources.review import Review -from sugar_network.resources.feedback import Feedback -from sugar_network.resources.solution import Solution -from sugar_network.resources.comment import Comment -from sugar_network.resources.implementation import Implementation +from sugar_network.model.user import User +from sugar_network.model.context import Context +from sugar_network.model.review import Review +from sugar_network.model.feedback import Feedback +from sugar_network.model.solution import Solution +from sugar_network.model.comment import Comment +from sugar_network.model.implementation import Implementation from sugar_network.toolkit import http diff --git a/tests/units/resources/context.py b/tests/units/model/context.py index d8c5628..d8c5628 100755 --- a/tests/units/resources/context.py +++ b/tests/units/model/context.py diff --git a/tests/units/resources/implementation.py b/tests/units/model/implementation.py index 176051c..a3afd4b 100755 --- a/tests/units/resources/implementation.py +++ b/tests/units/model/implementation.py @@ -8,64 +8,61 @@ import xapian from __init__ import tests from sugar_network import db -from sugar_network.db.router import Router, route -from sugar_network.resources import implementation -from sugar_network.resources.volume import Volume -from sugar_network.resources.implementation import _encode_version, Implementation -from sugar_network.node.commands import NodeCommands +from sugar_network.model import implementation +from sugar_network.model.implementation import _fmt_version, Implementation from sugar_network.client import IPCClient from sugar_network.toolkit import http, coroutine class ImplementationTest(tests.Test): - def test_encode_version(self): + def test_fmt_version(self): self.assertEqual( xapian.sortable_serialise(eval('1''0000''0000''5''000')), - _encode_version('1')) + _fmt_version('1')) self.assertEqual( xapian.sortable_serialise(eval('1''0002''0000''5''000')), - _encode_version('1.2')) + _fmt_version('1.2')) self.assertEqual( xapian.sortable_serialise(eval('1''0020''0300''5''000')), - _encode_version('1.20.300')) + _fmt_version('1.20.300')) self.assertEqual( xapian.sortable_serialise(eval('1''0020''0300''5''000')), - _encode_version('1.20.300.4444')) + _fmt_version('1.20.300.4444')) self.assertEqual( xapian.sortable_serialise(eval('1''9999''0000''5''000')), - _encode_version('10001.99999.10000')) + _fmt_version('10001.99999.10000')) self.assertEqual( xapian.sortable_serialise(eval('1''0000''0000''3''000')), - _encode_version('1-pre')) + _fmt_version('1-pre')) self.assertEqual( xapian.sortable_serialise(eval('1''0000''0000''4''000')), - _encode_version('1-rc')) + _fmt_version('1-rc')) self.assertEqual( xapian.sortable_serialise(eval('1''0000''0000''5''000')), - _encode_version('1-')) + _fmt_version('1-')) self.assertEqual( xapian.sortable_serialise(eval('1''0000''0000''6''000')), - _encode_version('1-r')) + _fmt_version('1-r')) self.assertEqual( xapian.sortable_serialise(eval('1''0000''0000''3''001')), - _encode_version('1-pre1')) + _fmt_version('1-pre1')) self.assertEqual( xapian.sortable_serialise(eval('1''0000''0000''4''002')), - _encode_version('1-rc2')) + _fmt_version('1-rc2')) self.assertEqual( xapian.sortable_serialise(eval('1''0000''0000''6''003')), - _encode_version('1-r3')) + _fmt_version('1-r3')) self.assertEqual( xapian.sortable_serialise(eval('1''0000''0000''6''000')), - _encode_version('1-r-2-3')) + _fmt_version('1-r-2-3')) self.assertEqual( xapian.sortable_serialise(eval('1''0000''0000''6''001')), - _encode_version('1-r1.2-3')) + _fmt_version('1-r1.2-3')) def test_WrongAuthor(self): self.start_online_client() diff --git a/tests/units/resources/review.py b/tests/units/model/review.py index c5cd30a..0e50f5a 100755 --- a/tests/units/resources/review.py +++ b/tests/units/model/review.py @@ -4,11 +4,11 @@ from __init__ import tests from sugar_network.client import Client -from sugar_network.resources.user import User -from sugar_network.resources.context import Context -from sugar_network.resources.review import Review -from sugar_network.resources.artifact import Artifact -from sugar_network.resources.implementation import Implementation +from sugar_network.model.user import User +from sugar_network.model.context import Context +from sugar_network.model.review import Review +from sugar_network.model.artifact import Artifact +from sugar_network.model.implementation import Implementation class ReviewTest(tests.Test): diff --git a/tests/units/model/routes.py b/tests/units/model/routes.py new file mode 100755 index 0000000..dd5bcb3 --- /dev/null +++ b/tests/units/model/routes.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# sugar-lint: disable + +import os +import json +import time +from email.utils import formatdate +from os.path import exists + +from __init__ import tests, src_root + +from sugar_network import db, model +from sugar_network.model.user import User +from sugar_network.toolkit.router import Router +from sugar_network.toolkit import coroutine + + +class RoutesTest(tests.Test): + + def test_StaticFiles(self): + router = Router(model.Routes()) + local_path = src_root + '/sugar_network/static/httpdocs/images/missing.png' + + response = [] + reply = router({ + 'PATH_INFO': '/static/images/missing.png', + 'REQUEST_METHOD': 'GET', + }, + lambda status, headers: response.extend([status, dict(headers)])) + result = file(local_path).read() + self.assertEqual(result, ''.join([i for i in reply])) + self.assertEqual([ + '200 OK', + { + 'last-modified': formatdate(os.stat(local_path).st_mtime, localtime=False, usegmt=True), + 'content-length': str(len(result)), + 'content-type': 'image/png', + 'content-disposition': 'attachment; filename="missing.png"', + } + ], + response) + + def test_Subscribe(self): + + class Document(db.Resource): + + @db.indexed_property(slot=1) + def prop(self, value): + return value + + routes = model.Routes() + volume = db.Volume('db', [Document], routes.broadcast) + events = [] + + def read_events(): + for event in routes.subscribe(event='!commit'): + if not event.strip(): + continue + assert event.startswith('data: ') + assert event.endswith('\n\n') + event = json.loads(event[6:]) + events.append(event) + + job = coroutine.spawn(read_events) + coroutine.dispatch() + volume['document'].create({'guid': 'guid', 'prop': 'value1'}) + coroutine.dispatch() + volume['document'].update('guid', {'prop': 'value2'}) + coroutine.dispatch() + volume['document'].delete('guid') + coroutine.dispatch() + volume['document'].commit() + coroutine.sleep(.5) + job.kill() + + self.assertEqual([ + {'guid': 'guid', 'resource': 'document', 'event': 'create'}, + {'guid': 'guid', 'resource': 'document', 'event': 'update'}, + {'guid': 'guid', 'event': 'delete', 'resource': u'document'}, + ], + events) + + def test_SubscribeWithPong(self): + routes = model.Routes() + for event in routes.subscribe(ping=True): + break + self.assertEqual('data: {"event": "pong"}\n\n', event) + + + +if __name__ == '__main__': + tests.main() diff --git a/tests/units/resources/solution.py b/tests/units/model/solution.py index a56ccfa..dafb5a7 100755 --- a/tests/units/resources/solution.py +++ b/tests/units/model/solution.py @@ -4,11 +4,11 @@ from __init__ import tests from sugar_network.client import Client -from sugar_network.resources.user import User -from sugar_network.resources.context import Context -from sugar_network.resources.feedback import Feedback -from sugar_network.resources.solution import Solution -from sugar_network.resources.implementation import Implementation +from sugar_network.model.user import User +from sugar_network.model.context import Context +from sugar_network.model.feedback import Feedback +from sugar_network.model.solution import Solution +from sugar_network.model.implementation import Implementation class SolutionTest(tests.Test): diff --git a/tests/units/node/__main__.py b/tests/units/node/__main__.py index 4ab7386..ac37315 100644 --- a/tests/units/node/__main__.py +++ b/tests/units/node/__main__.py @@ -2,7 +2,6 @@ from __init__ import tests -from auth import * from downloads import * from files import * from master import * diff --git a/tests/units/node/auth.py b/tests/units/node/auth.py deleted file mode 100755 index f445f3c..0000000 --- a/tests/units/node/auth.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env python -# sugar-lint: disable - -import os -import json - -from __init__ import tests - -from sugar_network import db, client -from sugar_network.node import auth -from sugar_network.client import IPCClient, Client -from sugar_network.resources.user import User -from sugar_network.toolkit import http, enforce - - -class AuthTest(tests.Test): - - def test_Config(self): - self.touch(('authorization.conf', [ - '[user_1]', - 'role_1 = True', - '[user_2]', - 'role_2 = False', - ])) - - request = db.Request() - request.principal = 'user_1' - self.assertEqual(True, auth.try_validate(request, 'role_1')) - auth.validate(request, 'role_1') - - request.principal = 'user_2' - self.assertEqual(False, auth.try_validate(request, 'role_2')) - self.assertRaises(http.Forbidden, auth.validate, request, 'role_2') - - request.principal = 'user_3' - self.assertEqual(False, auth.try_validate(request, 'role_1')) - self.assertEqual(False, auth.try_validate(request, 'role_2')) - self.assertRaises(http.Forbidden, auth.validate, request, 'role_1') - self.assertRaises(http.Forbidden, auth.validate, request, 'role_2') - - def test_FullWriteForRoot(self): - conn = Client() - - self.start_master() - conn.post(['context'], { - 'guid': 'guid', - 'type': 'package', - 'title': 'title', - 'summary': 'summary', - 'description': 'description', - }) - self.assertNotEqual('probe', conn.get(['context', 'guid', 'title'])) - self.stop_nodes() - - self.touch(( - 'master/context/gu/guid/author', - json.dumps({"seqno": 1, "value": {"fake": {"role": 3}}}), - )) - - self.start_master() - self.assertRaises(http.Forbidden, conn.put, ['context', 'guid'], {'title': 'probe'}) - - self.touch(('authorization.conf', [ - '[%s]' % tests.UID, - 'root = True', - ])) - auth.reset() - conn.put(['context', 'guid'], {'title': 'probe'}) - self.assertEqual('probe', conn.get(['context', 'guid', 'title'])) - - def test_Anonymous(self): - conn = http.Client(client.api_url.value) - - props = {'guid': 'guid', - 'type': 'package', - 'title': 'title', - 'summary': 'summary', - 'description': 'description', - } - self.start_master() - - self.assertRaises(RuntimeError, conn.post, ['context'], props) - - self.touch(('authorization.conf', [ - '[anonymous]', - 'user = True', - ])) - auth.reset() - conn.post(['context'], props) - self.assertEqual('title', conn.get(['context', 'guid', 'title'])) - self.assertEqual([], conn.get(['context', 'guid', 'author'])) - - self.stop_nodes() - self.touch(( - 'master/context/gu/guid/author', - json.dumps({"seqno": 1, "value": {"fake": {"role": 3}}}), - )) - self.start_master() - - auth.reset() - self.assertRaises(http.Forbidden, conn.put, ['context', 'guid'], {'title': 'probe'}) - - self.touch(('authorization.conf', [ - '[anonymous]', - 'user = True', - 'root = True', - ])) - auth.reset() - conn.put(['context', 'guid'], {'title': 'probe'}) - self.assertEqual('probe', conn.get(['context', 'guid', 'title'])) - self.assertEqual([{'name': 'fake', 'role': 3}], conn.get(['context', 'guid', 'author'])) - - def test_LiveUpdate(self): - conn = http.Client(client.api_url.value) - - props = {'guid': 'guid', - 'type': 'package', - 'title': 'title', - 'summary': 'summary', - 'description': 'description', - } - self.start_master() - - self.touch(('authorization.conf', '')) - os.utime('authorization.conf', (1, 1)) - self.assertRaises(RuntimeError, conn.post, ['context'], props) - - self.touch(('authorization.conf', [ - '[anonymous]', - 'user = True', - ])) - os.utime('authorization.conf', (2, 2)) - conn.post(['context'], props) - self.assertEqual([], conn.get(['context', 'guid', 'author'])) - - self.touch(('authorization.conf', '')) - os.utime('authorization.conf', (3, 3)) - self.assertRaises(RuntimeError, conn.post, ['context'], props) - - def test_DefaultAuthorization(self): - - class Document(db.Document): - - @db.document_command(method='GET', cmd='probe1', - mime_type='application/json') - def probe1(cls, directory): - return 'ok1' - - @db.document_command(method='GET', cmd='probe2', - permissions=db.ACCESS_AUTH, mime_type='application/json') - def probe2(cls, directory): - return 'ok2' - - self.start_master([User, Document]) - conn = Client() - - guid = conn.post(['document'], {}) - self.assertEqual('ok1', conn.get(['document', guid], cmd='probe1')) - self.assertEqual('ok2', conn.get(['document', guid], cmd='probe2')) - - -if __name__ == '__main__': - tests.main() diff --git a/tests/units/node/files.py b/tests/units/node/files.py index 2676aa9..111b7a8 100755 --- a/tests/units/node/files.py +++ b/tests/units/node/files.py @@ -11,7 +11,6 @@ from cStringIO import StringIO from __init__ import tests from sugar_network import db, toolkit -from sugar_network.toolkit import util from sugar_network.node import files @@ -27,14 +26,14 @@ class FilesTest(tests.Test): return str(self.uuid) def test_Index_Populate(self): - seqno = util.Seqno('seqno') + seqno = toolkit.Seqno('seqno') seeder = files.Index('files', 'index', seqno) os.utime('files', (1, 1)) assert seeder.sync() assert not seeder.sync() - in_seq = util.Sequence([[1, None]]) + in_seq = toolkit.Sequence([[1, None]]) self.assertEqual( [{'op': 'commit', 'sequence': []}], [i for i in seeder.diff(in_seq)]) @@ -48,7 +47,7 @@ class FilesTest(tests.Test): os.utime('files', (1, 1)) assert not seeder.sync() - in_seq = util.Sequence([[1, None]]) + in_seq = toolkit.Sequence([[1, None]]) self.assertEqual( [{'op': 'commit', 'sequence': []}], [i for i in seeder.diff(in_seq)]) @@ -59,7 +58,7 @@ class FilesTest(tests.Test): os.utime('files', (2, 2)) assert seeder.sync() - in_seq = util.Sequence([[1, None]]) + in_seq = toolkit.Sequence([[1, None]]) self.assertEqual(sorted([ {'op': 'commit', 'sequence': [[1, 3]]}, {'op': 'update', 'blob_size': 1, 'blob': '1', 'path': '1'}, @@ -79,14 +78,14 @@ class FilesTest(tests.Test): json.load(file('index'))) assert not seeder.sync() - in_seq = util.Sequence([[4, None]]) + in_seq = toolkit.Sequence([[4, None]]) self.assertEqual( [{'op': 'commit', 'sequence': []}], [i for i in seeder.diff(in_seq)]) self.assertEqual(3, seqno.value) def test_Index_SelectiveDiff(self): - seqno = util.Seqno('seqno') + seqno = toolkit.Seqno('seqno') seeder = files.Index('files', 'index', seqno) self.touch(('files/1', '1')) @@ -96,7 +95,7 @@ class FilesTest(tests.Test): self.touch(('files/5', '5')) self.utime('files', 1) - in_seq = util.Sequence([[2, 2], [4, 10], [20, None]]) + in_seq = toolkit.Sequence([[2, 2], [4, 10], [20, None]]) self.assertEqual(sorted([ {'op': 'commit', 'sequence': [[2, 5]]}, {'op': 'update', 'blob_size': 1, 'blob': '2', 'path': '2'}, @@ -106,7 +105,7 @@ class FilesTest(tests.Test): sorted(files_diff(seeder, in_seq))) def test_Index_PartialDiff(self): - seqno = util.Seqno('seqno') + seqno = toolkit.Seqno('seqno') seeder = files.Index('files', 'index', seqno) self.touch(('files/1', '1')) @@ -114,7 +113,7 @@ class FilesTest(tests.Test): self.touch(('files/3', '3')) self.utime('files', 1) - in_seq = util.Sequence([[1, None]]) + in_seq = toolkit.Sequence([[1, None]]) diff = seeder.diff(in_seq) record = next(diff) record['blob'] = ''.join([i for i in record['blob']]) @@ -133,7 +132,7 @@ class FilesTest(tests.Test): self.assertRaises(StopIteration, diff.next) def test_Index_diff_Stretch(self): - seqno = util.Seqno('seqno') + seqno = toolkit.Seqno('seqno') seeder = files.Index('files', 'index', seqno) self.touch(('files/1', '1')) @@ -141,7 +140,7 @@ class FilesTest(tests.Test): self.touch(('files/3', '3')) self.utime('files', 1) - in_seq = util.Sequence([[1, 1], [3, None]]) + in_seq = toolkit.Sequence([[1, 1], [3, None]]) diff = seeder.diff(in_seq) record = next(diff) record['blob'] = ''.join([i for i in record['blob']]) @@ -153,7 +152,7 @@ class FilesTest(tests.Test): self.assertRaises(StopIteration, diff.next) def test_Index_diff_DoNotStretchContinuesPacket(self): - seqno = util.Seqno('seqno') + seqno = toolkit.Seqno('seqno') seeder = files.Index('files', 'index', seqno) self.touch(('files/1', '1')) @@ -161,8 +160,8 @@ class FilesTest(tests.Test): self.touch(('files/3', '3')) self.utime('files', 1) - in_seq = util.Sequence([[1, 1], [3, None]]) - diff = seeder.diff(in_seq, util.Sequence([[1, 1]])) + in_seq = toolkit.Sequence([[1, 1], [3, None]]) + diff = seeder.diff(in_seq, toolkit.Sequence([[1, 1]])) record = next(diff) record['blob'] = ''.join([i for i in record['blob']]) self.assertEqual({'op': 'update', 'blob_size': 1, 'blob': '1', 'path': '1'}, record) @@ -173,7 +172,7 @@ class FilesTest(tests.Test): self.assertRaises(StopIteration, diff.next) def test_Index_DiffUpdatedFiles(self): - seqno = util.Seqno('seqno') + seqno = toolkit.Seqno('seqno') seeder = files.Index('files', 'index', seqno) self.touch(('files/1', '1')) @@ -182,7 +181,7 @@ class FilesTest(tests.Test): self.utime('files', 1) os.utime('files', (1, 1)) - for __ in seeder.diff(util.Sequence([[1, None]])): + for __ in seeder.diff(toolkit.Sequence([[1, None]])): pass self.assertEqual(3, seqno.value) @@ -190,7 +189,7 @@ class FilesTest(tests.Test): self.assertEqual( [{'op': 'commit', 'sequence': []}], - [i for i in seeder.diff(util.Sequence([[4, None]]))]) + [i for i in seeder.diff(toolkit.Sequence([[4, None]]))]) self.assertEqual(3, seqno.value) os.utime('files', (3, 3)) @@ -199,7 +198,7 @@ class FilesTest(tests.Test): {'op': 'update', 'blob_size': 1, 'blob': '2', 'path': '2'}, {'op': 'commit', 'sequence': [[4, 4]]}, ]), - sorted(files_diff(seeder, util.Sequence([[4, None]])))) + sorted(files_diff(seeder, toolkit.Sequence([[4, None]])))) self.assertEqual(4, seqno.value) os.utime('files/1', (4, 4)) @@ -211,7 +210,7 @@ class FilesTest(tests.Test): {'op': 'update', 'blob_size': 1, 'blob': '3', 'path': '3'}, {'op': 'commit', 'sequence': [[5, 6]]}, ]), - sorted(files_diff(seeder, util.Sequence([[5, None]])))) + sorted(files_diff(seeder, toolkit.Sequence([[5, None]])))) self.assertEqual(6, seqno.value) self.assertEqual(sorted([ @@ -220,11 +219,11 @@ class FilesTest(tests.Test): {'op': 'update', 'blob_size': 1, 'blob': '3', 'path': '3'}, {'op': 'commit', 'sequence': [[1, 6]]}, ]), - sorted(files_diff(seeder, util.Sequence([[1, None]])))) + sorted(files_diff(seeder, toolkit.Sequence([[1, None]])))) self.assertEqual(6, seqno.value) def test_Index_DiffCreatedFiles(self): - seqno = util.Seqno('seqno') + seqno = toolkit.Seqno('seqno') seeder = files.Index('files', 'index', seqno) self.touch(('files/1', '1')) @@ -233,7 +232,7 @@ class FilesTest(tests.Test): self.utime('files', 1) os.utime('files', (1, 1)) - for __ in seeder.diff(util.Sequence([[1, None]])): + for __ in seeder.diff(toolkit.Sequence([[1, None]])): pass self.assertEqual(3, seqno.value) @@ -243,7 +242,7 @@ class FilesTest(tests.Test): self.assertEqual( [{'op': 'commit', 'sequence': []}], - [i for i in seeder.diff(util.Sequence([[4, None]]))]) + [i for i in seeder.diff(toolkit.Sequence([[4, None]]))]) self.assertEqual(3, seqno.value) os.utime('files/4', (2, 2)) @@ -253,7 +252,7 @@ class FilesTest(tests.Test): {'op': 'update', 'blob_size': 1, 'blob': '4', 'path': '4'}, {'op': 'commit', 'sequence': [[4, 4]]}, ]), - sorted(files_diff(seeder, util.Sequence([[4, None]])))) + sorted(files_diff(seeder, toolkit.Sequence([[4, None]])))) self.assertEqual(4, seqno.value) self.touch(('files/5', '5')) @@ -267,11 +266,11 @@ class FilesTest(tests.Test): {'op': 'update', 'blob_size': 1, 'blob': '6', 'path': '6'}, {'op': 'commit', 'sequence': [[5, 6]]}, ]), - sorted(files_diff(seeder, util.Sequence([[5, None]])))) + sorted(files_diff(seeder, toolkit.Sequence([[5, None]])))) self.assertEqual(6, seqno.value) def test_Index_DiffDeletedFiles(self): - seqno = util.Seqno('seqno') + seqno = toolkit.Seqno('seqno') seeder = files.Index('files', 'index', seqno) self.touch(('files/1', '1')) @@ -280,7 +279,7 @@ class FilesTest(tests.Test): self.utime('files', 1) os.utime('files', (1, 1)) - for __ in seeder.diff(util.Sequence([[1, None]])): + for __ in seeder.diff(toolkit.Sequence([[1, None]])): pass self.assertEqual(3, seqno.value) @@ -294,7 +293,7 @@ class FilesTest(tests.Test): {'op': 'delete', 'path': '2'}, {'op': 'commit', 'sequence': [[1, 4]]}, ]), - sorted(files_diff(seeder, util.Sequence([[1, None]])))) + sorted(files_diff(seeder, toolkit.Sequence([[1, None]])))) self.assertEqual(4, seqno.value) os.unlink('files/1') @@ -308,7 +307,7 @@ class FilesTest(tests.Test): {'op': 'delete', 'path': '3'}, {'op': 'commit', 'sequence': [[1, 6]]}, ]), - sorted([i for i in seeder.diff(util.Sequence([[1, None]]))])) + sorted([i for i in seeder.diff(toolkit.Sequence([[1, None]]))])) self.assertEqual(6, seqno.value) assert not seeder.sync() @@ -318,7 +317,7 @@ class FilesTest(tests.Test): {'op': 'delete', 'path': '3'}, {'op': 'commit', 'sequence': [[1, 6]]}, ]), - sorted([i for i in seeder.diff(util.Sequence([[1, None]]))])) + sorted([i for i in seeder.diff(toolkit.Sequence([[1, None]]))])) self.assertEqual(6, seqno.value) def test_merge_Updated(self): diff --git a/tests/units/node/master.py b/tests/units/node/master.py index 242e159..f5dbc1b 100755 --- a/tests/units/node/master.py +++ b/tests/units/node/master.py @@ -153,7 +153,7 @@ class MasterTest(tests.Test): events = [] def read_events(): for event in ipc.subscribe(): - if event.get('document') == 'implementation': + if event.get('resource') == 'implementation': events.append(event) job = coroutine.spawn(read_events) @@ -174,7 +174,7 @@ class MasterTest(tests.Test): }) coroutine.sleep(.5) self.assertEqual([ - {'event': 'populate', 'document': 'implementation', 'mtime': int(os.stat('master/implementation/index/mtime').st_mtime)}, + {'event': 'populate', 'resource': 'implementation', 'mtime': int(os.stat('master/implementation/index/mtime').st_mtime)}, ], events) self.assertEqual({ @@ -189,7 +189,7 @@ class MasterTest(tests.Test): events = [] def read_events(): for event in ipc.subscribe(): - if event.get('document') == 'implementation': + if event.get('resource') == 'implementation': events.append(event) job = coroutine.spawn(read_events) @@ -205,7 +205,7 @@ class MasterTest(tests.Test): ipc.put(['context', guid, 'dependencies'], ['foo']) coroutine.sleep(.1) self.assertEqual([ - {'event': 'populate', 'document': 'implementation', 'mtime': int(os.stat('master/implementation/index/mtime').st_mtime)}, + {'event': 'populate', 'resource': 'implementation', 'mtime': int(os.stat('master/implementation/index/mtime').st_mtime)}, ], events) diff --git a/tests/units/node/node.py b/tests/units/node/node.py index a1bb46b..85dcff3 100755 --- a/tests/units/node/node.py +++ b/tests/units/node/node.py @@ -10,22 +10,22 @@ from os.path import exists from __init__ import tests -from sugar_network import db, node +from sugar_network import db, node, model from sugar_network.client import Client from sugar_network.toolkit import http, coroutine from sugar_network.toolkit.rrd import Rrd from sugar_network.node import stats_user, stats_node, obs -from sugar_network.node.commands import NodeCommands -from sugar_network.node.master import MasterCommands -from sugar_network.resources.volume import Volume, Resource -from sugar_network.resources.user import User -from sugar_network.resources.context import Context -from sugar_network.resources.implementation import Implementation -from sugar_network.resources.review import Review -from sugar_network.resources.feedback import Feedback -from sugar_network.resources.artifact import Artifact -from sugar_network.resources.solution import Solution -from sugar_network.resources.user import User +from sugar_network.node.routes import NodeRoutes +from sugar_network.node.master import MasterRoutes +from sugar_network.model.user import User +from sugar_network.model.context import Context +from sugar_network.model.implementation import Implementation +from sugar_network.model.review import Review +from sugar_network.model.feedback import Feedback +from sugar_network.model.artifact import Artifact +from sugar_network.model.solution import Solution +from sugar_network.model.user import User +from sugar_network.toolkit.router import Router, Request, Response, fallbackroute, Blob, ACL, route class NodeTest(tests.Test): @@ -37,8 +37,8 @@ class NodeTest(tests.Test): stats_user.stats_user_rras.value = ['RRA:AVERAGE:0.5:1:100'] def test_UserStats(self): - volume = Volume('db') - cp = NodeCommands('guid', volume) + volume = db.Volume('db', model.RESOURCES) + cp = NodeRoutes('guid', volume) call(cp, method='POST', document='user', principal=tests.UID, content={ 'name': 'user', @@ -110,8 +110,8 @@ class NodeTest(tests.Test): for i in range(100): rrd['user'].put({'total': i}, ts + i) - volume = Volume('db', [User, Context, Review, Feedback, Solution, Artifact]) - cp = NodeCommands('guid', volume) + volume = db.Volume('db', model.RESOURCES) + cp = NodeRoutes('guid', volume) self.assertEqual({ 'user': [ @@ -134,10 +134,11 @@ class NodeTest(tests.Test): cp.stats(ts, ts + 12, 3, ['user.total'])) def test_HandleDeletes(self): - volume = Volume('db') - cp = NodeCommands('guid', volume) + volume = db.Volume('db', model.RESOURCES) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': tests.PUBKEY}) + cp = NodeRoutes('guid', volume) - guid = call(cp, method='POST', document='context', principal='principal', content={ + guid = call(cp, method='POST', document='context', principal=tests.UID, content={ 'type': 'activity', 'title': 'title', 'summary': 'summary', @@ -154,43 +155,44 @@ class NodeTest(tests.Test): call(cp, method='GET', document='context', guid=guid, reply=['guid', 'title', 'layer'])) self.assertEqual(['public'], volume['context'].get(guid)['layer']) + def subscribe(): + for event in cp.subscribe(): + events.append(json.loads(event[6:])) events = [] - volume.connect(lambda event: events.append(event)) - call(cp, method='DELETE', document='context', guid=guid, principal='principal') + coroutine.spawn(subscribe) coroutine.dispatch() + call(cp, method='DELETE', document='context', guid=guid, principal=tests.UID) + coroutine.dispatch() self.assertRaises(http.NotFound, call, cp, method='GET', document='context', guid=guid, reply=['guid', 'title']) self.assertEqual(['deleted'], volume['context'].get(guid)['layer']) - self.assertEqual([ - {'event': 'delete', 'document': 'context', 'guid': guid}, - {'event': 'commit', 'document': 'context', 'mtime': int(os.stat('db/context/index/mtime').st_mtime)}, - ], - events) + self.assertEqual({'event': 'delete', 'resource': 'context', 'guid': guid}, events[0]) def test_SimulateDeleteEvents(self): - volume = Volume('db') - cp = NodeCommands('guid', volume) + volume = db.Volume('db', model.RESOURCES) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': tests.PUBKEY}) + cp = NodeRoutes('guid', volume) - guid = call(cp, method='POST', document='context', principal='principal', content={ + guid = call(cp, method='POST', document='context', principal=tests.UID, content={ 'type': 'activity', 'title': 'title', 'summary': 'summary', 'description': 'description', }) + def subscribe(): + for event in cp.subscribe(): + events.append(json.loads(event[6:])) events = [] - volume.connect(lambda event: events.append(event)) - call(cp, method='PUT', document='context', guid=guid, principal='principal', content={'layer': ['deleted']}) + coroutine.spawn(subscribe) coroutine.dispatch() - self.assertEqual([ - {'event': 'delete', 'document': 'context', 'guid': guid}, - {'event': 'commit', 'document': 'context', 'mtime': int(os.stat('db/context/index/mtime').st_mtime)}, - ], - events) + call(cp, method='PUT', document='context', guid=guid, principal=tests.UID, content={'layer': ['deleted']}) + coroutine.dispatch() + self.assertEqual({'event': 'delete', 'resource': 'context', 'guid': guid}, events[0]) def test_RegisterUser(self): - cp = NodeCommands('guid', Volume('db', [User])) + cp = NodeRoutes('guid', db.Volume('db', [User])) guid = call(cp, method='POST', document='user', principal='fake', content={ 'name': 'user', @@ -203,50 +205,57 @@ class NodeTest(tests.Test): self.assertEqual('user', call(cp, method='GET', document='user', guid=tests.UID, prop='name')) def test_UnauthorizedCommands(self): + volume = db.Volume('db', model.RESOURCES) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': tests.PUBKEY}) - class Document(Resource): + class Routes(NodeRoutes): - @db.document_command(method='GET', cmd='probe1', - permissions=db.ACCESS_AUTH) + @route('GET', [None, None], cmd='probe1', acl=ACL.AUTH) def probe1(self, directory): pass - @db.document_command(method='GET', cmd='probe2') + @route('GET', [None, None], cmd='probe2') def probe2(self, directory): pass - cp = NodeCommands('guid', Volume('db', [User, Document])) - guid = call(cp, method='POST', document='document', principal='user', content={}) + class Document(db.Resource): + pass + + cp = Routes('guid', db.Volume('db', [User, Document])) + guid = call(cp, method='POST', document='document', principal=tests.UID, content={}) + self.assertRaises(http.Unauthorized, call, cp, method='GET', cmd='probe1', document='document', guid=guid) - call(cp, method='GET', cmd='probe1', document='document', guid=guid, principal='user') + call(cp, method='GET', cmd='probe1', document='document', guid=guid, principal=tests.UID) call(cp, method='GET', cmd='probe2', document='document', guid=guid) def test_ForbiddenCommands(self): - class Document(Resource): + class Routes(NodeRoutes): - @db.document_command(method='GET', cmd='probe1', - permissions=db.ACCESS_AUTHOR) + @route('GET', [None, None], cmd='probe1', acl=ACL.AUTHOR) def probe1(self): pass - @db.document_command(method='GET', cmd='probe2') + @route('GET', [None, None], cmd='probe2') def probe2(self): pass - class User(db.Document): + class Document(db.Resource): pass - cp = NodeCommands('guid', Volume('db', [User, Document])) - guid = call(cp, method='POST', document='document', principal='principal', content={}) + volume = db.Volume('db', [User, Document]) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': tests.PUBKEY}) + cp = Routes('guid', volume) + + guid = call(cp, method='POST', document='document', principal=tests.UID, content={}) self.assertRaises(http.Forbidden, call, cp, method='GET', cmd='probe1', document='document', guid=guid) - self.assertRaises(http.Forbidden, call, cp, method='GET', cmd='probe1', document='document', guid=guid, principal='fake') - call(cp, method='GET', cmd='probe1', document='document', guid=guid, principal='principal') + self.assertRaises(http.Unauthorized, call, cp, method='GET', cmd='probe1', document='document', guid=guid, principal='fake') + call(cp, method='GET', cmd='probe1', document='document', guid=guid, principal=tests.UID) call(cp, method='GET', cmd='probe2', document='document', guid=guid) def test_ForbiddenCommandsForUserResource(self): - cp = NodeCommands('guid', Volume('db', [User])) + cp = NodeRoutes('guid', db.Volume('db', [User])) call(cp, method='POST', document='user', principal='fake', content={ 'name': 'user1', @@ -258,39 +267,134 @@ class NodeTest(tests.Test): self.assertEqual('user1', call(cp, method='GET', document='user', guid=tests.UID, prop='name')) self.assertRaises(http.Unauthorized, call, cp, method='PUT', document='user', guid=tests.UID, content={'name': 'user2'}) - self.assertRaises(http.Forbidden, call, cp, method='PUT', document='user', guid=tests.UID, principal='fake', content={'name': 'user2'}) + self.assertRaises(http.Unauthorized, call, cp, method='PUT', document='user', guid=tests.UID, principal='fake', content={'name': 'user2'}) call(cp, method='PUT', document='user', guid=tests.UID, principal=tests.UID, content={'name': 'user2'}) self.assertEqual('user2', call(cp, method='GET', document='user', guid=tests.UID, prop='name')) + def test_authorize_Config(self): + self.touch(('authorization.conf', [ + '[%s]' % tests.UID, + 'root = True', + ])) + + class Routes(NodeRoutes): + + @route('PROBE', acl=ACL.SUPERUSER) + def probe(self): + return 'ok' + + volume = db.Volume('db', [User]) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': tests.PUBKEY}) + volume['user'].create({'guid': tests.UID2, 'name': 'test', 'color': '', 'pubkey': tests.PUBKEY2}) + cp = Routes('guid', volume) + + self.assertRaises(http.Forbidden, call, cp, method='PROBE') + self.assertRaises(http.Forbidden, call, cp, method='PROBE', principal=tests.UID2) + self.assertEqual('ok', call(cp, method='PROBE', principal=tests.UID)) + + def test_authorize_FullWriteForRoot(self): + self.touch(('authorization.conf', [ + '[%s]' % tests.UID2, + 'root = True', + ])) + + class Routes(NodeRoutes): + + @route('PROBE', [None, None], acl=ACL.AUTHOR) + def probe(self): + pass + + class Document(db.Resource): + pass + + volume = db.Volume('db', [User, Document]) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': tests.PUBKEY}) + volume['user'].create({'guid': tests.UID2, 'name': 'user', 'color': '', 'pubkey': tests.PUBKEY2}) + cp = Routes('guid', volume) + + guid = call(cp, method='POST', document='document', principal=tests.UID, content={}) + + call(cp, 'PROBE', document='document', guid=guid, principal=tests.UID) + call(cp, 'PROBE', document='document', guid=guid, principal=tests.UID2) + + def test_authorize_LiveConfigUpdates(self): + + class Routes(NodeRoutes): + + @route('PROBE', acl=ACL.SUPERUSER) + def probe(self): + pass + + volume = db.Volume('db', [User]) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': tests.PUBKEY}) + cp = Routes('guid', volume) + + self.assertRaises(http.Forbidden, call, cp, 'PROBE', principal=tests.UID) + self.touch(('authorization.conf', [ + '[%s]' % tests.UID, + 'root = True', + ])) + call(cp, 'PROBE', principal=tests.UID) + + def test_authorize_Anonymous(self): + + class Routes(NodeRoutes): + + @route('PROBE1', acl=ACL.AUTH) + def probe1(self, request): + pass + + @route('PROBE2', acl=ACL.SUPERUSER) + def probe2(self, request): + pass + + volume = db.Volume('db', [User]) + cp = Routes('guid', volume) + + self.assertRaises(http.Unauthorized, call, cp, 'PROBE1') + self.assertRaises(http.Forbidden, call, cp, 'PROBE2') + + self.touch(('authorization.conf', [ + '[anonymous]', + 'user = True', + 'root = True', + ])) + call(cp, 'PROBE1') + call(cp, 'PROBE2') + def test_SetUser(self): - cp = NodeCommands('guid', Volume('db')) + volume = db.Volume('db', model.RESOURCES) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': tests.PUBKEY}) + cp = NodeRoutes('guid', volume) - guid = call(cp, method='POST', document='context', principal='principal', content={ + guid = call(cp, method='POST', document='context', principal=tests.UID, content={ 'type': 'activity', 'title': 'title', 'summary': 'summary', 'description': 'description', }) self.assertEqual( - [{'name': 'principal', 'role': 2}], + [{'guid': tests.UID, 'name': 'user', 'role': 3}], call(cp, method='GET', document='context', guid=guid, prop='author')) def test_find_MaxLimit(self): - cp = NodeCommands('guid', Volume('db')) + volume = db.Volume('db', model.RESOURCES) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': tests.PUBKEY}) + cp = NodeRoutes('guid', volume) - call(cp, method='POST', document='context', principal='principal', content={ + call(cp, method='POST', document='context', principal=tests.UID, content={ 'type': 'activity', 'title': 'title1', 'summary': 'summary', 'description': 'description', }) - call(cp, method='POST', document='context', principal='principal', content={ + call(cp, method='POST', document='context', principal=tests.UID, content={ 'type': 'activity', 'title': 'title2', 'summary': 'summary', 'description': 'description', }) - call(cp, method='POST', document='context', principal='principal', content={ + call(cp, method='POST', document='context', principal=tests.UID, content={ 'type': 'activity', 'title': 'title3', 'summary': 'summary', @@ -305,10 +409,11 @@ class NodeTest(tests.Test): self.assertEqual(1, len(call(cp, method='GET', document='context', limit=1024)['result'])) def test_DeletedDocuments(self): - volume = Volume('db') - cp = NodeCommands('guid', volume) + volume = db.Volume('db', model.RESOURCES) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': tests.PUBKEY}) + cp = NodeRoutes('guid', volume) - guid = call(cp, method='POST', document='context', principal='principal', content={ + guid = call(cp, method='POST', document='context', principal=tests.UID, content={ 'type': 'activity', 'title': 'title1', 'summary': 'summary', @@ -325,9 +430,10 @@ class NodeTest(tests.Test): def test_CreateGUID(self): # TODO Temporal security hole, see TODO - volume2 = Volume('db2') - cp2 = MasterCommands('guid', volume2) - call(cp2, method='POST', document='context', principal='principal', content={ + volume = db.Volume('db', model.RESOURCES) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': tests.PUBKEY}) + cp = NodeRoutes('guid', volume) + call(cp, method='POST', document='context', principal=tests.UID, content={ 'guid': 'foo', 'type': 'activity', 'title': 'title', @@ -336,12 +442,14 @@ class NodeTest(tests.Test): }) self.assertEqual( {'guid': 'foo', 'title': 'title'}, - call(cp2, method='GET', document='context', guid='foo', reply=['guid', 'title'])) + call(cp, method='GET', document='context', guid='foo', reply=['guid', 'title'])) def test_CreateMalformedGUID(self): - cp = MasterCommands('guid', Volume('db2')) + volume = db.Volume('db', model.RESOURCES) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': tests.PUBKEY}) + cp = MasterRoutes('guid', volume) - self.assertRaises(RuntimeError, call, cp, method='POST', document='context', principal='principal', content={ + self.assertRaises(RuntimeError, call, cp, method='POST', document='context', principal=tests.UID, content={ 'guid': '!?', 'type': 'activity', 'title': 'title', @@ -350,16 +458,18 @@ class NodeTest(tests.Test): }) def test_FailOnExistedGUID(self): - cp = MasterCommands('guid', Volume('db2')) + volume = db.Volume('db', model.RESOURCES) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': tests.PUBKEY}) + cp = MasterRoutes('guid', volume) - guid = call(cp, method='POST', document='context', principal='principal', content={ + guid = call(cp, method='POST', document='context', principal=tests.UID, content={ 'type': 'activity', 'title': 'title', 'summary': 'summary', 'description': 'description', }) - self.assertRaises(RuntimeError, call, cp, method='POST', document='context', principal='principal', content={ + self.assertRaises(RuntimeError, call, cp, method='POST', document='context', principal=tests.UID, content={ 'guid': guid, 'type': 'activity', 'title': 'title', @@ -550,16 +660,23 @@ class NodeTest(tests.Test): self.assertEqual(len(activity_info), data.get('unpack_size')) - - - - -def call(cp, principal=None, content=None, **kwargs): - request = db.Request(**kwargs) - request.principal = principal +def call(routes, method, document=None, guid=None, prop=None, principal=None, cmd=None, content=None, **kwargs): + path = [] + if document: + path.append(document) + if guid: + path.append(guid) + if prop: + path.append(prop) + request = Request(method=method, path=path) + request.update(kwargs) + request.cmd = cmd request.content = content request.environ = {'HTTP_HOST': '127.0.0.1'} - return cp.call(request, db.Response()) + if principal: + request.environ['HTTP_X_SN_LOGIN'] = principal + router = Router(routes) + return router.call(request, Response()) if __name__ == '__main__': diff --git a/tests/units/node/stats_node.py b/tests/units/node/stats_node.py index 607a380..2fa9446 100755 --- a/tests/units/node/stats_node.py +++ b/tests/units/node/stats_node.py @@ -5,25 +5,19 @@ import time from __init__ import tests -from sugar_network.toolkit.rrd import Rrd +from sugar_network import db, model from sugar_network.node.stats_node import stats_node_step, Sniffer -from sugar_network.resources.user import User -from sugar_network.resources.context import Context -from sugar_network.resources.implementation import Implementation -from sugar_network.resources.review import Review -from sugar_network.resources.feedback import Feedback -from sugar_network.resources.artifact import Artifact -from sugar_network.resources.solution import Solution -from sugar_network.resources.volume import Volume +from sugar_network.toolkit.rrd import Rrd +from sugar_network.toolkit.router import Request class StatsTest(tests.Test): - def test_DoNotLogAnonymouses(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) + def ___test_DoNotLogAnonymouses(self): + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) - request = db.Request(method='GET', document='context', guid='guid') + request = Request(method='GET', path=['context', 'guid']) stats.log(request) self.assertEqual(0, stats._stats['context'].viewed) @@ -32,20 +26,20 @@ class StatsTest(tests.Test): self.assertEqual(1, stats._stats['context'].viewed) def test_DoNotLogCmds(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) - request = db.Request(method='GET', document='context', guid='guid', cmd='probe') + request = Request(method='GET', path=['context', 'guid'], cmd='probe') request.principal = 'user' stats.log(request) self.assertEqual(0, stats._stats['context'].viewed) - del request['cmd'] + request.cmd = None stats.log(request) self.assertEqual(1, stats._stats['context'].viewed) def test_InitializeTotals(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) self.assertEqual(0, stats._stats['user'].total) @@ -74,10 +68,10 @@ class StatsTest(tests.Test): self.assertEqual(1, stats._stats['artifact'].total) def test_POSTs(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) - request = db.Request(method='POST', document='context') + request = Request(method='POST', path=['context']) request.principal = 'user' stats.log(request) stats.log(request) @@ -88,10 +82,10 @@ class StatsTest(tests.Test): self.assertEqual(0, stats._stats['context'].deleted) def test_PUTs(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) - request = db.Request(method='PUT', document='context', guid='guid') + request = Request(method='PUT', path=['context', 'guid']) request.principal = 'user' stats.log(request) stats.log(request) @@ -102,10 +96,10 @@ class StatsTest(tests.Test): self.assertEqual(0, stats._stats['context'].deleted) def test_DELETEs(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) - request = db.Request(method='DELETE', document='context') + request = Request(method='DELETE', path=['context']) request.principal = 'user' stats.log(request) stats.log(request) @@ -116,29 +110,29 @@ class StatsTest(tests.Test): self.assertEqual(3, stats._stats['context'].deleted) def test_GETs(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) - request = db.Request(method='GET', document='user') + request = Request(method='GET', path=['user']) request.principal = 'user' stats.log(request) self.assertEqual(0, stats._stats['user'].viewed) def test_GETsDocument(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) - request = db.Request(method='GET', document='user', guid='user') + request = Request(method='GET', path=['user', 'user']) request.principal = 'user' stats.log(request) self.assertEqual(1, stats._stats['user'].viewed) def test_FeedbackSolutions(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) volume['feedback'].create({'guid': 'guid', 'context': 'context', 'type': 'idea', 'title': '', 'content': ''}) - request = db.Request(method='PUT', document='feedback', guid='guid') + request = Request(method='PUT', path=['feedback', 'guid']) request.principal = 'user' request.content = {} stats.log(request) @@ -162,51 +156,51 @@ class StatsTest(tests.Test): self.assertEqual(0, stats._stats['feedback'].solutions) def test_Comments(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) volume['solution'].create({'guid': 'solution', 'context': 'context', 'feedback': 'feedback', 'content': ''}) volume['feedback'].create({'guid': 'feedback', 'context': 'context', 'type': 'idea', 'title': '', 'content': ''}) volume['review'].create({'guid': 'review', 'context': 'context', 'title': '', 'content': '', 'rating': 5}) - request = db.Request(method='POST', document='comment') + request = Request(method='POST', path=['comment']) request.principal = 'user' request.content = {'solution': 'solution'} stats.log(request) self.assertEqual(1, stats._stats['solution'].commented) - request = db.Request(method='POST', document='comment') + request = Request(method='POST', path=['comment']) request.principal = 'user' request.content = {'feedback': 'feedback'} stats.log(request) self.assertEqual(1, stats._stats['feedback'].commented) - request = db.Request(method='POST', document='comment') + request = Request(method='POST', path=['comment']) request.principal = 'user' request.content = {'review': 'review'} stats.log(request) self.assertEqual(1, stats._stats['review'].commented) def test_Reviewes(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''}) - request = db.Request(method='POST', document='review') + request = Request(method='POST', path=['review']) request.principal = 'user' request.content = {'context': 'context', 'rating': 0} stats.log(request) self.assertEqual(1, stats._stats['context'].reviewed) self.assertEqual(0, stats._stats['artifact'].reviewed) - request = db.Request(method='POST', document='review') + request = Request(method='POST', path=['review']) request.principal = 'user' request.content = {'context': 'context', 'artifact': '', 'rating': 0} stats.log(request) self.assertEqual(2, stats._stats['context'].reviewed) self.assertEqual(0, stats._stats['artifact'].reviewed) - request = db.Request(method='POST', document='review') + request = Request(method='POST', path=['review']) request.principal = 'user' request.content = {'artifact': 'artifact', 'rating': 0} stats.log(request) @@ -214,55 +208,55 @@ class StatsTest(tests.Test): self.assertEqual(1, stats._stats['artifact'].reviewed) def test_ContextDownloaded(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact, Implementation]) + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) volume['implementation'].create({'guid': 'implementation', 'context': 'context', 'license': 'GPLv3', 'version': '1', 'date': 0, 'stability': 'stable', 'notes': ''}) - request = db.Request(method='GET', document='implementation', guid='implementation', prop='fake') + request = Request(method='GET', path=['implementation', 'implementation', 'fake']) request.principal = 'user' stats.log(request) self.assertEqual(0, stats._stats['context'].downloaded) - request = db.Request(method='GET', document='implementation', guid='implementation', prop='data') + request = Request(method='GET', path=['implementation', 'implementation', 'data']) request.principal = 'user' stats.log(request) self.assertEqual(1, stats._stats['context'].downloaded) def test_ContextReleased(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact, Implementation]) + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) - request = db.Request(method='POST', document='implementation') + request = Request(method='POST', path=['implementation']) request.principal = 'user' request.content = {'context': 'context'} stats.log(request) self.assertEqual(1, stats._stats['context'].released) def test_ContextFailed(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact, Implementation]) + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) - request = db.Request(method='POST', document='report') + request = Request(method='POST', path=['report']) request.principal = 'user' request.content = {'context': 'context'} stats.log(request) self.assertEqual(1, stats._stats['context'].failed) def test_ContextActive(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact, Implementation]) + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) - request = db.Request(method='PUT', document='context', guid='1') + request = Request(method='PUT', path=['context', '1']) request.principal = 'user' stats.log(request) self.assertEqual( ['1'], stats._stats['context'].active.keys()) - request = db.Request(method='GET', document='artifact', context='2') + request = Request(method='GET', path=['artifact'], context='2') request.principal = 'user' stats.log(request) self.assertEqual( @@ -270,7 +264,7 @@ class StatsTest(tests.Test): stats._stats['context'].active.keys()) volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': '3', 'title': '', 'description': ''}) - request = db.Request(method='GET', document='review', artifact='artifact') + request = Request(method='GET', path=['review'], artifact='artifact') request.principal = 'user' stats.log(request) self.assertEqual( @@ -278,21 +272,21 @@ class StatsTest(tests.Test): sorted(stats._stats['context'].active.keys())) volume['feedback'].create({'guid': 'feedback', 'context': '4', 'type': 'idea', 'title': '', 'content': ''}) - request = db.Request(method='GET', document='solution', feedback='feedback') + request = Request(method='GET', path=['solution'], feedback='feedback') request.principal = 'user' stats.log(request) self.assertEqual( ['1', '2', '3', '4'], sorted(stats._stats['context'].active.keys())) - request = db.Request(method='GET', document='context', guid='5') + request = Request(method='GET', path=['context', '5']) request.principal = 'user' stats.log(request) self.assertEqual( ['1', '2', '3', '4', '5'], sorted(stats._stats['context'].active.keys())) - request = db.Request(method='POST', document='report') + request = Request(method='POST', path=['report']) request.principal = 'user' request.content = {'context': '6'} stats.log(request) @@ -301,7 +295,7 @@ class StatsTest(tests.Test): sorted(stats._stats['context'].active.keys())) volume['solution'].create({'guid': 'solution', 'context': '7', 'feedback': 'feedback', 'content': ''}) - request = db.Request(method='POST', document='comment') + request = Request(method='POST', path=['comment']) request.principal = 'user' request.content = {'solution': 'solution'} stats.log(request) @@ -310,10 +304,10 @@ class StatsTest(tests.Test): sorted(stats._stats['context'].active.keys())) def test_UserActive(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact, Implementation]) + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) - request = db.Request(method='GET', document='user') + request = Request(method='GET', path=['user']) request.principal = '1' stats.log(request) self.assertEqual( @@ -323,7 +317,7 @@ class StatsTest(tests.Test): set([]), stats._stats['user'].effective) - request = db.Request(method='POST', document='user') + request = Request(method='POST', path=['user']) request.principal = '2' stats.log(request) self.assertEqual( @@ -334,17 +328,17 @@ class StatsTest(tests.Test): stats._stats['user'].effective) def test_ArtifactDownloaded(self): - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) + volume = db.Volume('local', model.RESOURCES) stats = Sniffer(volume) volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''}) - request = db.Request(method='GET', document='artifact', guid='artifact', prop='fake') + request = Request(method='GET', path=['artifact', 'artifact', 'fake']) request.principal = 'user' stats.log(request) self.assertEqual(0, stats._stats['artifact'].viewed) self.assertEqual(0, stats._stats['artifact'].downloaded) - request = db.Request(method='GET', document='artifact', guid='artifact', prop='data') + request = Request(method='GET', path=['artifact', 'artifact', 'data']) request.principal = 'user' stats.log(request) self.assertEqual(0, stats._stats['artifact'].viewed) @@ -352,7 +346,7 @@ class StatsTest(tests.Test): def test_Commit(self): stats_node_step.value = 1 - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) + volume = db.Volume('local', model.RESOURCES) volume['user'].create({'guid': 'user', 'name': 'user', 'color': '', 'pubkey': ''}) volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) volume['review'].create({'guid': 'review', 'context': 'context', 'title': '', 'content': '', 'rating': 5}) @@ -361,22 +355,22 @@ class StatsTest(tests.Test): volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''}) stats = Sniffer(volume) - request = db.Request(method='GET', document='user', guid='user') + request = Request(method='GET', path=['user', 'user']) request.principal = 'user' stats.log(request) - request = db.Request(method='GET', document='context', guid='context') + request = Request(method='GET', path=['context', 'context']) request.principal = 'user' stats.log(request) - request = db.Request(method='GET', document='review', guid='review') + request = Request(method='GET', path=['review', 'review']) request.principal = 'user' stats.log(request) - request = db.Request(method='GET', document='feedback', guid='feedback') + request = Request(method='GET', path=['feedback', 'feedback']) request.principal = 'user' stats.log(request) - request = db.Request(method='GET', document='solution', guid='solution') + request = Request(method='GET', path=['solution', 'solution']) request.principal = 'user' stats.log(request) - request = db.Request(method='GET', document='artifact', guid='artifact') + request = Request(method='GET', path=['artifact', 'artifact']) request.principal = 'user' stats.log(request) @@ -474,11 +468,11 @@ class StatsTest(tests.Test): 'released': 0.0, })], ], - [[(db.name,) + i for i in db.get(db.first, db.last)] for db in Rrd('stats/node', 1)]) + [[(j.name,) + i for i in j.get(j.last, j.last)] for j in Rrd('stats/node', 1)]) def test_CommitContextStats(self): stats_node_step.value = 1 - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact, Implementation]) + volume = db.Volume('local', model.RESOURCES) volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) volume['implementation'].create({'guid': 'implementation', 'context': 'context', 'license': 'GPLv3', 'version': '1', 'date': 0, 'stability': 'stable', 'notes': ''}) @@ -488,14 +482,14 @@ class StatsTest(tests.Test): self.assertEqual(0, volume['context'].get('context')['rating']) stats = Sniffer(volume) - request = db.Request(method='GET', document='implementation', guid='implementation', prop='data') + request = Request(method='GET', path=['implementation', 'implementation', 'data']) request.principal = 'user' stats.log(request) - request = db.Request(method='POST', document='review') + request = Request(method='POST', path=['review']) request.principal = 'user' request.content = {'context': 'context', 'rating': 5} stats.log(request) - request = db.Request(method='POST', document='review') + request = Request(method='POST', path=['review']) request.principal = 'user' request.content = {'artifact': 'artifact', 'rating': 5} stats.log(request) @@ -510,10 +504,10 @@ class StatsTest(tests.Test): self.assertEqual(5, volume['context'].get('context')['rating']) stats = Sniffer(volume) - request = db.Request(method='GET', document='implementation', guid='implementation', prop='data') + request = Request(method='GET', path=['implementation', 'implementation', 'data']) request.principal = 'user' stats.log(request) - request = db.Request(method='POST', document='review') + request = Request(method='POST', path=['review']) request.principal = 'user' request.content = {'context': 'context', 'rating': 1} stats.log(request) @@ -524,7 +518,7 @@ class StatsTest(tests.Test): def test_CommitArtifactStats(self): stats_node_step.value = 1 - volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact, Implementation]) + volume = db.Volume('local', model.RESOURCES) volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''}) @@ -533,10 +527,10 @@ class StatsTest(tests.Test): self.assertEqual(0, volume['artifact'].get('artifact')['rating']) stats = Sniffer(volume) - request = db.Request(method='GET', document='artifact', guid='artifact', prop='data') + request = Request(method='GET', path=['artifact', 'artifact', 'data']) request.principal = 'user' stats.log(request) - request = db.Request(method='POST', document='review') + request = Request(method='POST', path=['review']) request.principal = 'user' request.content = {'artifact': 'artifact', 'rating': 5} stats.log(request) @@ -550,10 +544,10 @@ class StatsTest(tests.Test): self.assertEqual([1, 5], volume['artifact'].get('artifact')['reviews']) self.assertEqual(5, volume['artifact'].get('artifact')['rating']) - request = db.Request(method='GET', document='artifact', guid='artifact', prop='data') + request = Request(method='GET', path=['artifact', 'artifact', 'data']) request.principal = 'user' stats.log(request) - request = db.Request(method='POST', document='review') + request = Request(method='POST', path=['review']) request.principal = 'user' request.content = {'artifact': 'artifact', 'rating': 1} stats.log(request) diff --git a/tests/units/node/sync_master.py b/tests/units/node/sync_master.py index e137f6e..c946396 100755 --- a/tests/units/node/sync_master.py +++ b/tests/units/node/sync_master.py @@ -18,10 +18,11 @@ from __init__ import tests from sugar_network.db.directory import Directory from sugar_network import db, node, toolkit from sugar_network.node import sync -from sugar_network.node.master import MasterCommands -from sugar_network.resources.volume import Volume -from sugar_network.toolkit import coroutine, util +from sugar_network.node.master import MasterRoutes +from sugar_network.db.volume import Volume +from sugar_network.toolkit import coroutine from sugar_network.toolkit.rrd import Rrd +from sugar_network.toolkit.router import Response class statvfs(object): @@ -40,7 +41,7 @@ class SyncMasterTest(tests.Test): self.override(os, 'statvfs', lambda *args: statvfs()) statvfs.f_bfree = 999999999 - class Document(db.Document): + class Document(db.Resource): @db.indexed_property(slot=1, default='') def prop(self, value): @@ -48,7 +49,7 @@ class SyncMasterTest(tests.Test): node.files_root.value = 'sync' self.volume = Volume('master', [Document]) - self.master = MasterCommands('127.0.0.1:8888', self.volume) + self.master = MasterRoutes('127.0.0.1:8888', self.volume) def next_uuid(self): self.uuid += 1 @@ -58,7 +59,7 @@ class SyncMasterTest(tests.Test): request = Request() for chunk in sync.encode([ ('diff', None, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '1', 'diff': { 'guid': {'value': '1', 'mtime': 1}, 'ctime': {'value': 1, 'mtime': 1}, @@ -78,7 +79,7 @@ class SyncMasterTest(tests.Test): response.seek(0) self.assertEqual([ ({'packet': 'ack', 'ack': [[1, 1]], 'src': '127.0.0.1:8888', 'sequence': [[1, 1]], 'dst': None}, []), - ({'packet': 'diff', 'src': '127.0.0.1:8888'}, [{'document': 'document'}, {'commit': []}]), + ({'packet': 'diff', 'src': '127.0.0.1:8888'}, [{'resource': 'document'}, {'commit': []}]), ], [(packet.props, [i for i in packet]) for packet in sync.decode(response)]) @@ -86,7 +87,7 @@ class SyncMasterTest(tests.Test): for chunk in sync.encode([ ('pull', {'sequence': [[1, None]]}, None), ('diff', None, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '2', 'diff': { 'guid': {'value': '2', 'mtime': 2}, 'ctime': {'value': 2, 'mtime': 2}, @@ -106,7 +107,7 @@ class SyncMasterTest(tests.Test): self.assertEqual([ ({'packet': 'ack', 'ack': [[2, 2]], 'src': '127.0.0.1:8888', 'sequence': [[2, 2]], 'dst': None}, []), ({'packet': 'diff', 'src': '127.0.0.1:8888'}, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '1', 'diff': { 'guid': {'value': '1', 'mtime': 1}, 'ctime': {'value': 1, 'mtime': 1}, @@ -143,7 +144,7 @@ class SyncMasterTest(tests.Test): request = Request() for chunk in sync.package_encode([ ('diff', None, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '1', 'diff': { 'guid': {'value': '1', 'mtime': 1}, 'ctime': {'value': 1, 'mtime': 1}, @@ -161,7 +162,7 @@ class SyncMasterTest(tests.Test): request.content_stream.write(chunk) request.content_stream.seek(0) - response = db.Response() + response = Response() reply = StringIO() for chunk in self.master.push(request, response): reply.write(chunk) @@ -186,7 +187,7 @@ class SyncMasterTest(tests.Test): ('pull', {'sequence': [[1, None]]}, None), ('files_pull', {'sequence': [[1, None]]}, None), ('diff', None, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '2', 'diff': { 'guid': {'value': '2', 'mtime': 2}, 'ctime': {'value': 2, 'mtime': 2}, @@ -204,7 +205,7 @@ class SyncMasterTest(tests.Test): request.content_stream.write(chunk) request.content_stream.seek(0) - response = db.Response() + response = Response() reply = StringIO() for chunk in self.master.push(request, response): reply.write(chunk) @@ -235,7 +236,7 @@ class SyncMasterTest(tests.Test): ], dst='127.0.0.1:8888'): request.content_stream.write(chunk) request.content_stream.seek(0) - response = db.Response() + response = Response() reply = StringIO() for chunk in self.master.push(request, response): reply.write(chunk) @@ -259,7 +260,7 @@ class SyncMasterTest(tests.Test): ], dst='127.0.0.1:8888'): request.content_stream.write(chunk) request.content_stream.seek(0) - response = db.Response() + response = Response() reply = StringIO() for chunk in self.master.push(request, response): reply.write(chunk) @@ -281,7 +282,7 @@ class SyncMasterTest(tests.Test): for chunk in sync.package_encode([ ('pull', {'sequence': [[1, None]]}, None), ('diff', None, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '1', 'diff': { 'guid': {'value': '1', 'mtime': 1}, 'ctime': {'value': 1, 'mtime': 1}, @@ -294,7 +295,7 @@ class SyncMasterTest(tests.Test): request.content_stream.write(chunk) request.content_stream.seek(0) - response = db.Response() + response = Response() reply = StringIO() for chunk in self.master.push(request, response): reply.write(chunk) @@ -322,7 +323,7 @@ class SyncMasterTest(tests.Test): request = Request() request.environ['HTTP_COOKIE'] = 'sugar_network_pull=%s' % \ base64.b64encode(json.dumps([('pull', None, [[1, None]]), ('files_pull', None, [[1, None]])])) - response = db.Response() + response = Response() self.assertEqual(None, self.master.pull(request, response)) self.assertEqual([ 'sugar_network_pull=%s; Max-Age=3600; HttpOnly' % \ @@ -336,14 +337,14 @@ class SyncMasterTest(tests.Test): request = Request() request.environ['HTTP_COOKIE'] = response.get('set-cookie')[0] - response = db.Response() + response = Response() reply = StringIO() for chunk in self.master.pull(request, response): reply.write(chunk) reply.seek(0) self.assertEqual([ ({'packet': 'diff'}, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '1', 'diff': { 'prop': {'value': '1', 'mtime': 0}, 'guid': {'value': '1', 'mtime': 0}, @@ -371,7 +372,7 @@ class SyncMasterTest(tests.Test): request = Request() request.environ['HTTP_COOKIE'] = response.get('set-cookie')[0] - response = db.Response() + response = Response() reply = StringIO() for chunk in self.master.pull(request, response): reply.write(chunk) @@ -401,7 +402,7 @@ class SyncMasterTest(tests.Test): request = Request() request.environ['HTTP_COOKIE'] = 'sugar_network_pull=%s' % \ base64.b64encode(json.dumps([('pull', None, [[1, None]])])) - response = db.Response() + response = Response() self.assertEqual(None, self.master.pull(request, response)) self.assertEqual([ 'sugar_network_pull=%s; Max-Age=3600; HttpOnly' % \ @@ -416,7 +417,7 @@ class SyncMasterTest(tests.Test): request = Request() request.environ['HTTP_COOKIE'] = response.get('set-cookie')[0] - response = db.Response() + response = Response() self.assertEqual(None, self.master.pull(request, response)) self.assertEqual([ 'sugar_network_pull=unset_sugar_network_pull; Max-Age=0; HttpOnly', @@ -438,7 +439,7 @@ class SyncMasterTest(tests.Test): } request = Request() - response = db.Response() + response = Response() self.assertEqual(None, self.master.pull(request, response)) self.assertEqual([ 'sugar_network_pull=%s; Max-Age=3600; HttpOnly' % \ @@ -452,7 +453,7 @@ class SyncMasterTest(tests.Test): request = Request() request.environ['HTTP_COOKIE'] = response.get('set-cookie')[0] - response = db.Response() + response = Response() reply = StringIO() for chunk in self.master.pull(request, response): reply.write(chunk) @@ -472,7 +473,7 @@ class SyncMasterTest(tests.Test): request = Request() request.environ['HTTP_COOKIE'] = response.get('set-cookie')[0] - response = db.Response() + response = Response() reply = StringIO() for chunk in self.master.pull(request, response): reply.write(chunk) @@ -496,7 +497,7 @@ class SyncMasterTest(tests.Test): ('pull', {'src': '2', 'sequence': [[2, None]], 'layer': '2'}, None), ('pull', {'src': '2', 'sequence': [[22, None]], 'layer': '2'}, None), ('diff', {'src': '3'}, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '1', 'diff': { 'guid': {'value': '1', 'mtime': 1}, 'ctime': {'value': 1, 'mtime': 1}, @@ -506,7 +507,7 @@ class SyncMasterTest(tests.Test): {'commit': [[1, 1]]}, ]), ('diff', {'src': '3'}, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '2', 'diff': { 'guid': {'value': '2', 'mtime': 2}, 'ctime': {'value': 2, 'mtime': 2}, @@ -519,7 +520,7 @@ class SyncMasterTest(tests.Test): request.content_stream.write(chunk) request.content_stream.seek(0) - response = db.Response() + response = Response() reply = StringIO() for chunk in self.master.push(request, response): reply.write(chunk) @@ -548,7 +549,7 @@ class SyncMasterTest(tests.Test): 'sugar_network_pull': base64.b64encode(json.dumps([('pull', None, [[1, None]])])), 'sugar_network_sent': base64.b64encode(json.dumps({'slave': [[2, 2]]})), } - response = db.Response() + response = Response() self.assertEqual(None, self.master.pull(request, response)) self.assertEqual([ 'sugar_network_pull=%s; Max-Age=3600; HttpOnly' % \ @@ -568,7 +569,7 @@ class SyncMasterTest(tests.Test): reply.seek(0) self.assertEqual([ ({'packet': 'diff'}, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '1', 'diff': { 'prop': {'value': '1', 'mtime': 0}, 'guid': {'value': '1', 'mtime': 0}, @@ -596,7 +597,7 @@ class SyncMasterTest(tests.Test): 'sugar_network_pull': base64.b64encode(json.dumps([('pull', None, [[1, None]])])), 'sugar_network_sent': base64.b64encode(json.dumps({'slave': [[2, 2]], 'other': []})), } - response = db.Response() + response = Response() self.assertEqual(None, self.master.pull(request, response)) self.assertEqual([ 'sugar_network_pull=%s; Max-Age=3600; HttpOnly' % \ @@ -616,7 +617,7 @@ class SyncMasterTest(tests.Test): reply.seek(0) self.assertEqual([ ({'packet': 'diff'}, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '1', 'diff': { 'prop': {'value': '1', 'mtime': 0}, 'guid': {'value': '1', 'mtime': 0}, diff --git a/tests/units/node/sync_offline.py b/tests/units/node/sync_offline.py index 752ff93..f3b7111 100755 --- a/tests/units/node/sync_offline.py +++ b/tests/units/node/sync_offline.py @@ -15,9 +15,9 @@ from sugar_network import db, node, toolkit from sugar_network.toolkit.rrd import Rrd from sugar_network.client import api_url from sugar_network.node import sync, stats_user, files_root -from sugar_network.node.slave import SlaveCommands -from sugar_network.resources.volume import Volume -from sugar_network.toolkit import coroutine, util +from sugar_network.node.slave import SlaveRoutes +from sugar_network.db import Volume +from sugar_network.toolkit import coroutine class statvfs(object): @@ -44,12 +44,12 @@ class SyncOfflineTest(tests.Test): def test_FailOnFullDump(self): - class Document(db.Document): + class Document(db.Resource): pass volume = Volume('node', [Document]) - util.ensure_key('node/key') - cp = SlaveCommands('node/key', volume) + toolkit.ensure_key('node/key') + cp = SlaveRoutes('node/key', volume) node.sync_layers.value = None self.assertRaises(RuntimeError, cp.offline_sync, tests.tmpdir + '/mnt') @@ -62,12 +62,12 @@ class SyncOfflineTest(tests.Test): def test_Export(self): - class Document(db.Document): + class Document(db.Resource): pass volume = Volume('node', [Document]) - util.ensure_key('node/key') - cp = SlaveCommands('node/key', volume) + toolkit.ensure_key('node/key') + cp = SlaveRoutes('node/key', volume) stats_user.stats_user.value = True volume['document'].create({'guid': '1', 'prop': 'value1', 'ctime': 1, 'mtime': 1}) @@ -83,7 +83,7 @@ class SyncOfflineTest(tests.Test): self.assertEqual([ ({'packet': 'diff', 'src': cp.guid, 'dst': '127.0.0.1:8888', 'api_url': 'http://127.0.0.1:8888', 'session': '1', 'filename': '2.sneakernet'}, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '1', 'diff': { 'guid': {'value': '1', 'mtime': 0}, 'ctime': {'value': 1, 'mtime': 0}, @@ -111,15 +111,15 @@ class SyncOfflineTest(tests.Test): def test_ContinuesExport(self): payload = ''.join([str(uuid.uuid4()) for i in xrange(5000)]) - class Document(db.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def prop(self, value): return value volume = Volume('node', [Document]) - util.ensure_key('node/key') - cp = SlaveCommands('node/key', volume) + toolkit.ensure_key('node/key') + cp = SlaveRoutes('node/key', volume) stats_user.stats_user.value = True volume['document'].create({'guid': '1', 'prop': payload, 'ctime': 1, 'mtime': 1}) @@ -136,7 +136,7 @@ class SyncOfflineTest(tests.Test): self.assertEqual([ ({'packet': 'diff', 'src': cp.guid, 'dst': '127.0.0.1:8888', 'api_url': 'http://127.0.0.1:8888', 'session': '1', 'filename': '2.sneakernet'}, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '1', 'diff': { 'guid': {'value': '1', 'mtime': 0}, 'ctime': {'value': 1, 'mtime': 0}, @@ -156,7 +156,7 @@ class SyncOfflineTest(tests.Test): self.assertEqual([ ({'packet': 'diff', 'src': cp.guid, 'dst': '127.0.0.1:8888', 'api_url': 'http://127.0.0.1:8888', 'session': '1', 'filename': '3.sneakernet'}, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '2', 'diff': { 'guid': {'value': '2', 'mtime': 0}, 'ctime': {'value': 2, 'mtime': 0}, @@ -179,7 +179,7 @@ class SyncOfflineTest(tests.Test): self.assertEqual([ ({'packet': 'diff', 'src': cp.guid, 'dst': '127.0.0.1:8888', 'api_url': 'http://127.0.0.1:8888', 'session': '4', 'filename': '5.sneakernet'}, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '1', 'diff': { 'guid': {'value': '1', 'mtime': 0}, 'ctime': {'value': 1, 'mtime': 0}, @@ -205,19 +205,19 @@ class SyncOfflineTest(tests.Test): sorted([(packet.props, [i for i in packet]) for packet in sync.sneakernet_decode('3')])) def test_Import(self): - class Document(db.Document): + class Document(db.Resource): pass volume = Volume('node', [Document]) - util.ensure_key('node/key') - cp = SlaveCommands('node/key', volume) + toolkit.ensure_key('node/key') + cp = SlaveRoutes('node/key', volume) stats_user.stats_user.value = True files_root.value = 'files' ts = int(time.time()) sync.sneakernet_encode([ ('diff', {'src': '127.0.0.1:8888'}, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '1', 'diff': { 'guid': {'value': '1', 'mtime': 0}, 'ctime': {'value': 1, 'mtime': 0}, diff --git a/tests/units/node/sync_online.py b/tests/units/node/sync_online.py index 9f978e8..c8bf5f8 100755 --- a/tests/units/node/sync_online.py +++ b/tests/units/node/sync_online.py @@ -7,16 +7,16 @@ from os.path import exists from __init__ import tests -from sugar_network import db -from sugar_network.db.router import Router +from sugar_network import db, toolkit from sugar_network.client import Client, api_url from sugar_network.node import sync, stats_user, files_root -from sugar_network.node.master import MasterCommands -from sugar_network.node.slave import SlaveCommands -from sugar_network.resources.volume import Volume -from sugar_network.resources.user import User -from sugar_network.resources.feedback import Feedback -from sugar_network.toolkit import util, coroutine +from sugar_network.node.master import MasterRoutes +from sugar_network.node.slave import SlaveRoutes +from sugar_network.db.volume import Volume +from sugar_network.model.user import User +from sugar_network.model.feedback import Feedback +from sugar_network.toolkit.router import Router +from sugar_network.toolkit import coroutine class SyncOnlineTest(tests.Test): @@ -42,14 +42,14 @@ class SyncOnlineTest(tests.Test): files_root.value = 'master/files' self.master_volume = Volume('master', [User, Document]) - self.master_server = coroutine.WSGIServer(('127.0.0.1', 9000), Router(MasterCommands('127.0.0.1:9000', self.master_volume))) + self.master_server = coroutine.WSGIServer(('127.0.0.1', 9000), Router(MasterRoutes('127.0.0.1:9000', self.master_volume))) coroutine.spawn(self.master_server.serve_forever) coroutine.dispatch() files_root.value = 'slave/files' self.slave_volume = Volume('slave', [User, Document]) - util.ensure_key('slave/node') - self.slave_server = coroutine.WSGIServer(('127.0.0.1', 9001), Router(SlaveCommands('slave/node', self.slave_volume))) + toolkit.ensure_key('slave/node') + self.slave_server = coroutine.WSGIServer(('127.0.0.1', 9001), Router(SlaveRoutes('slave/node', self.slave_volume))) coroutine.spawn(self.slave_server.serve_forever) coroutine.dispatch() diff --git a/tests/units/node/volume.py b/tests/units/node/volume.py index 15772e1..9a3a113 100755 --- a/tests/units/node/volume.py +++ b/tests/units/node/volume.py @@ -9,18 +9,11 @@ from cStringIO import StringIO from __init__ import tests -from sugar_network import db +from sugar_network import db, toolkit, model from sugar_network.node.volume import diff, merge from sugar_network.node.stats_node import stats_node_step, Sniffer -from sugar_network.node.commands import NodeCommands -from sugar_network.resources.user import User -from sugar_network.resources.volume import Volume, Resource -from sugar_network.resources.review import Review -from sugar_network.resources.context import Context -from sugar_network.resources.artifact import Artifact -from sugar_network.resources.feedback import Feedback -from sugar_network.resources.solution import Solution -from sugar_network.toolkit import util +from sugar_network.node.routes import NodeRoutes +from sugar_network.toolkit.router import Router, Request, Response, fallbackroute, Blob, ACL, route class VolumeTest(tests.Test): @@ -28,32 +21,36 @@ class VolumeTest(tests.Test): def setUp(self): tests.Test.setUp(self) self.override(time, 'time', lambda: 0) + self.override(NodeRoutes, 'authorize', lambda self, user, role: True) def test_diff(self): - class Document(db.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def prop(self, value): return value - volume = Volume('db', [Document]) - cp = NodeCommands('guid', volume) + volume = db.Volume('db', [Document]) + cp = NodeRoutes('guid', volume) - guid1 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'a'}) + guid1 = call(cp, method='POST', document='document', content={'prop': 'a'}) self.utime('db/document/%s/%s' % (guid1[:2], guid1), 1) - guid2 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'b'}) + guid2 = call(cp, method='POST', document='document', content={'prop': 'b'}) self.utime('db/document/%s/%s' % (guid2[:2], guid2), 2) - in_seq = util.Sequence([[1, None]]) + in_seq = toolkit.Sequence([[1, None]]) self.assertEqual([ - {'document': 'document'}, + {'resource': 'document'}, {'guid': guid1, 'diff': { 'guid': {'value': guid1, 'mtime': 1}, 'mtime': {'value': 0, 'mtime': 1}, 'ctime': {'value': 0, 'mtime': 1}, 'prop': {'value': 'a', 'mtime': 1}, + 'author': {'mtime': 1, 'value': {}}, + 'layer': {'mtime': 1, 'value': ['public']}, + 'tags': {'mtime': 1, 'value': []}, }, }, {'guid': guid2, @@ -62,6 +59,9 @@ class VolumeTest(tests.Test): 'mtime': {'value': 0, 'mtime': 2}, 'ctime': {'value': 0, 'mtime': 2}, 'prop': {'value': 'b', 'mtime': 2}, + 'author': {'mtime': 2, 'value': {}}, + 'layer': {'mtime': 2, 'value': ['public']}, + 'tags': {'mtime': 2, 'value': []}, }, }, {'commit': [[1, 2]]}, @@ -71,23 +71,23 @@ class VolumeTest(tests.Test): def test_diff_Partial(self): - class Document(db.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def prop(self, value): return value - volume = Volume('db', [Document]) - cp = NodeCommands('guid', volume) + volume = db.Volume('db', [Document]) + cp = NodeRoutes('guid', volume) - guid1 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'a'}) + guid1 = call(cp, method='POST', document='document', content={'prop': 'a'}) self.utime('db/document/%s/%s' % (guid1[:2], guid1), 1) - guid2 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'b'}) + guid2 = call(cp, method='POST', document='document', content={'prop': 'b'}) self.utime('db/document/%s/%s' % (guid2[:2], guid2), 2) - in_seq = util.Sequence([[1, None]]) + in_seq = toolkit.Sequence([[1, None]]) patch = diff(volume, in_seq) - self.assertEqual({'document': 'document'}, next(patch)) + self.assertEqual({'resource': 'document'}, next(patch)) self.assertEqual(guid1, next(patch)['guid']) self.assertEqual({'commit': []}, patch.throw(StopIteration())) try: @@ -97,7 +97,7 @@ class VolumeTest(tests.Test): pass patch = diff(volume, in_seq) - self.assertEqual({'document': 'document'}, next(patch)) + self.assertEqual({'resource': 'document'}, next(patch)) self.assertEqual(guid1, next(patch)['guid']) self.assertEqual(guid2, next(patch)['guid']) self.assertEqual({'commit': [[1, 1]]}, patch.throw(StopIteration())) @@ -109,29 +109,29 @@ class VolumeTest(tests.Test): def test_diff_Stretch(self): - class Document(db.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def prop(self, value): return value - volume = Volume('db', [Document]) - cp = NodeCommands('guid', volume) + volume = db.Volume('db', [Document]) + cp = NodeRoutes('guid', volume) - guid1 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'a'}) + guid1 = call(cp, method='POST', document='document', content={'prop': 'a'}) self.utime('db/document/%s/%s' % (guid1[:2], guid1), 1) - guid2 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'b'}) + guid2 = call(cp, method='POST', document='document', content={'prop': 'b'}) volume['document'].delete(guid2) - guid3 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'c'}) + guid3 = call(cp, method='POST', document='document', content={'prop': 'c'}) self.utime('db/document/%s/%s' % (guid3[:2], guid3), 2) - guid4 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'd'}) + guid4 = call(cp, method='POST', document='document', content={'prop': 'd'}) volume['document'].delete(guid4) - guid5 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'f'}) + guid5 = call(cp, method='POST', document='document', content={'prop': 'f'}) self.utime('db/document/%s/%s' % (guid5[:2], guid5), 2) - in_seq = util.Sequence([[1, None]]) + in_seq = toolkit.Sequence([[1, None]]) patch = diff(volume, in_seq) - self.assertEqual({'document': 'document'}, patch.send(None)) + self.assertEqual({'resource': 'document'}, patch.send(None)) self.assertEqual(guid1, patch.send(None)['guid']) self.assertEqual(guid3, patch.send(None)['guid']) self.assertEqual(guid5, patch.send(None)['guid']) @@ -143,7 +143,7 @@ class VolumeTest(tests.Test): pass patch = diff(volume, in_seq) - self.assertEqual({'document': 'document'}, patch.send(None)) + self.assertEqual({'resource': 'document'}, patch.send(None)) self.assertEqual(guid1, patch.send(None)['guid']) self.assertEqual(guid3, patch.send(None)['guid']) self.assertEqual(guid5, patch.send(None)['guid']) @@ -156,29 +156,29 @@ class VolumeTest(tests.Test): def test_diff_DoNotStretchContinuesPacket(self): - class Document(db.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def prop(self, value): return value - volume = Volume('db', [Document]) - cp = NodeCommands('guid', volume) + volume = db.Volume('db', [Document]) + cp = NodeRoutes('guid', volume) - guid1 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'a'}) + guid1 = call(cp, method='POST', document='document', content={'prop': 'a'}) volume['document'].delete(guid1) - guid2 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'b'}) + guid2 = call(cp, method='POST', document='document', content={'prop': 'b'}) volume['document'].delete(guid2) - guid3 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'c'}) + guid3 = call(cp, method='POST', document='document', content={'prop': 'c'}) self.utime('db/document/%s/%s' % (guid3[:2], guid3), 2) - guid4 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'd'}) + guid4 = call(cp, method='POST', document='document', content={'prop': 'd'}) volume['document'].delete(guid4) - guid5 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'f'}) + guid5 = call(cp, method='POST', document='document', content={'prop': 'f'}) self.utime('db/document/%s/%s' % (guid5[:2], guid5), 2) - in_seq = util.Sequence([[1, None]]) - patch = diff(volume, in_seq, util.Sequence([[1, 1]])) - self.assertEqual({'document': 'document'}, patch.send(None)) + in_seq = toolkit.Sequence([[1, None]]) + patch = diff(volume, in_seq, toolkit.Sequence([[1, 1]])) + self.assertEqual({'resource': 'document'}, patch.send(None)) self.assertEqual(guid3, patch.send(None)['guid']) self.assertEqual(guid5, patch.send(None)['guid']) self.assertEqual({'commit': [[1, 1], [3, 3], [5, 5]]}, patch.send(None)) @@ -190,32 +190,32 @@ class VolumeTest(tests.Test): def test_diff_TheSameInSeqForAllDocuments(self): - class Document1(db.Document): + class Document1(db.Resource): pass - class Document2(db.Document): + class Document2(db.Resource): pass - class Document3(db.Document): + class Document3(db.Resource): pass - volume = Volume('db', [Document1, Document2, Document3]) - cp = NodeCommands('guid', volume) + volume = db.Volume('db', [Document1, Document2, Document3]) + cp = NodeRoutes('guid', volume) - guid3 = call(cp, method='POST', document='document1', principal='principal', content={}) + guid3 = call(cp, method='POST', document='document1', content={}) self.utime('db/document/%s/%s' % (guid3[:2], guid3), 3) - guid2 = call(cp, method='POST', document='document2', principal='principal', content={}) + guid2 = call(cp, method='POST', document='document2', content={}) self.utime('db/document/%s/%s' % (guid2[:2], guid2), 2) - guid1 = call(cp, method='POST', document='document3', principal='principal', content={}) + guid1 = call(cp, method='POST', document='document3', content={}) self.utime('db/document/%s/%s' % (guid1[:2], guid1), 1) - in_seq = util.Sequence([[1, None]]) + in_seq = toolkit.Sequence([[1, None]]) patch = diff(volume, in_seq) - self.assertEqual({'document': 'document1'}, patch.send(None)) + self.assertEqual({'resource': 'document1'}, patch.send(None)) self.assertEqual(guid3, patch.send(None)['guid']) - self.assertEqual({'document': 'document2'}, patch.send(None)) + self.assertEqual({'resource': 'document2'}, patch.send(None)) self.assertEqual(guid2, patch.send(None)['guid']) - self.assertEqual({'document': 'document3'}, patch.send(None)) + self.assertEqual({'resource': 'document3'}, patch.send(None)) self.assertEqual(guid1, patch.send(None)['guid']) self.assertEqual({'commit': [[1, 3]]}, patch.send(None)) try: @@ -226,27 +226,27 @@ class VolumeTest(tests.Test): def test_merge_Create(self): - class Document1(db.Document): + class Document1(db.Resource): @db.indexed_property(slot=1) def prop(self, value): return value - class Document2(db.Document): + class Document2(db.Resource): pass self.touch(('db/seqno', '100')) - volume = Volume('db', [Document1, Document2]) + volume = db.Volume('db', [Document1, Document2]) records = [ - {'document': 'document1'}, + {'resource': 'document1'}, {'guid': '1', 'diff': { 'guid': {'value': '1', 'mtime': 1.0}, 'ctime': {'value': 2, 'mtime': 2.0}, 'mtime': {'value': 3, 'mtime': 3.0}, 'prop': {'value': '4', 'mtime': 4.0}, }}, - {'document': 'document2'}, + {'resource': 'document2'}, {'guid': '5', 'diff': { 'guid': {'value': '5', 'mtime': 5.0}, 'ctime': {'value': 6, 'mtime': 6.0}, @@ -273,20 +273,20 @@ class VolumeTest(tests.Test): def test_merge_Update(self): - class Document(db.Document): + class Document(db.Resource): @db.indexed_property(slot=1) def prop(self, value): return value self.touch(('db/seqno', '100')) - volume = Volume('db', [Document]) + volume = db.Volume('db', [Document]) volume['document'].create({'guid': '1', 'prop': '1', 'ctime': 1, 'mtime': 1}) for i in os.listdir('db/document/1/1'): os.utime('db/document/1/1/%s' % i, (2, 2)) records = [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '1', 'diff': {'prop': {'value': '2', 'mtime': 1.0}}}, {'commit': [[1, 1]]}, ] @@ -297,7 +297,7 @@ class VolumeTest(tests.Test): self.assertEqual(2, os.stat('db/document/1/1/prop').st_mtime) records = [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '1', 'diff': {'prop': {'value': '3', 'mtime': 2.0}}}, {'commit': [[2, 2]]}, ] @@ -308,7 +308,7 @@ class VolumeTest(tests.Test): self.assertEqual(2, os.stat('db/document/1/1/prop').st_mtime) records = [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '1', 'diff': {'prop': {'value': '4', 'mtime': 3.0}}}, {'commit': [[3, 3]]}, ] @@ -320,15 +320,15 @@ class VolumeTest(tests.Test): def test_merge_MultipleCommits(self): - class Document(db.Document): + class Document(db.Resource): pass self.touch(('db/seqno', '100')) - volume = Volume('db', [Document]) + volume = db.Volume('db', [Document]) def generator(): for i in [ - {'document': 'document'}, + {'resource': 'document'}, {'commit': [[1, 1]]}, {'guid': '1', 'diff': { 'guid': {'value': '1', 'mtime': 1.0}, @@ -346,18 +346,18 @@ class VolumeTest(tests.Test): def test_merge_UpdateReviewStats(self): stats_node_step.value = 1 - volume = Volume('db', [User, Context, Review, Feedback, Solution, Artifact]) - cp = NodeCommands('guid', volume) + volume = db.Volume('db', model.RESOURCES) + cp = NodeRoutes('guid', volume) stats = Sniffer(volume) - context = call(cp, method='POST', document='context', principal='principal', content={ + context = call(cp, method='POST', document='context', content={ 'guid': 'context', 'type': 'package', 'title': 'title', 'summary': 'summary', 'description': 'description', }) - artifact = call(cp, method='POST', document='artifact', principal='principal', content={ + artifact = call(cp, method='POST', document='artifact', content={ 'guid': 'artifact', 'type': 'instance', 'context': context, @@ -366,7 +366,7 @@ class VolumeTest(tests.Test): }) records = [ - {'document': 'review'}, + {'resource': 'review'}, {'guid': '1', 'diff': { 'guid': {'value': '1', 'mtime': 1.0}, 'ctime': {'value': 1, 'mtime': 1.0}, @@ -374,6 +374,9 @@ class VolumeTest(tests.Test): 'context': {'value': context, 'mtime': 1.0}, 'artifact': {'value': artifact, 'mtime': 4.0}, 'rating': {'value': 1, 'mtime': 1.0}, + 'author': {'mtime': 1, 'value': {}}, + 'layer': {'mtime': 1, 'value': ['public']}, + 'tags': {'mtime': 1, 'value': []}, }}, {'guid': '2', 'diff': { 'guid': {'value': '2', 'mtime': 2.0}, @@ -381,6 +384,9 @@ class VolumeTest(tests.Test): 'mtime': {'value': 2, 'mtime': 2.0}, 'context': {'value': context, 'mtime': 2.0}, 'rating': {'value': 2, 'mtime': 2.0}, + 'author': {'mtime': 2, 'value': {}}, + 'layer': {'mtime': 2, 'value': ['public']}, + 'tags': {'mtime': 2, 'value': []}, }}, {'commit': [[1, 2]]}, ] @@ -394,22 +400,22 @@ class VolumeTest(tests.Test): def test_diff_Blobs(self): - class Document(Resource): + class Document(db.Resource): @db.blob_property() def prop(self, value): return value - volume = Volume('db', [User, Document]) - cp = NodeCommands('guid', volume) + volume = db.Volume('db', [Document]) + cp = NodeRoutes('guid', volume) - guid = call(cp, method='POST', document='document', principal='principal', content={}) - call(cp, method='PUT', document='document', guid=guid, principal='principal', content={'prop': 'payload'}) + guid = call(cp, method='POST', document='document', content={}) + call(cp, method='PUT', document='document', guid=guid, content={'prop': 'payload'}) self.utime('db', 0) - patch = diff(volume, util.Sequence([[1, None]])) + patch = diff(volume, toolkit.Sequence([[1, None]])) self.assertEqual( - {'document': 'document'}, + {'resource': 'document'}, next(patch)) record = next(patch) self.assertEqual('payload', ''.join([i for i in record.pop('blob')])) @@ -426,7 +432,7 @@ class VolumeTest(tests.Test): self.assertEqual( {'guid': guid, 'diff': { 'guid': {'value': guid, 'mtime': 0}, - 'author': {'value': {'principal': {'order': 0, 'role': 2}}, 'mtime': 0}, + 'author': {'mtime': 0, 'value': {}}, 'layer': {'mtime': 0, 'value': ['public']}, 'tags': {'mtime': 0, 'value': []}, 'mtime': {'value': 0, 'mtime': 0}, @@ -434,9 +440,6 @@ class VolumeTest(tests.Test): }}, next(patch)) self.assertEqual( - {'document': 'user'}, - next(patch)) - self.assertEqual( {'commit': [[1, 2]]}, next(patch)) self.assertRaises(StopIteration, next, patch) @@ -445,25 +448,25 @@ class VolumeTest(tests.Test): url = 'http://src.sugarlabs.org/robots.txt' blob = urllib2.urlopen(url).read() - class Document(Resource): + class Document(db.Resource): @db.blob_property() def prop(self, value): return value - volume = Volume('db', [User, Document]) - cp = NodeCommands('guid', volume) + volume = db.Volume('db', [Document]) + cp = NodeRoutes('guid', volume) - guid = call(cp, method='POST', document='document', principal='principal', content={}) - call(cp, method='PUT', document='document', guid=guid, principal='principal', content={'prop': {'url': url}}) + guid = call(cp, method='POST', document='document', content={}) + call(cp, method='PUT', document='document', guid=guid, content={'prop': {'url': url}}) self.utime('db', 1) self.assertEqual([ - {'document': 'document'}, + {'resource': 'document'}, {'guid': guid, 'diff': { 'guid': {'value': guid, 'mtime': 1}, - 'author': {'value': {'principal': {'order': 0, 'role': 2}}, 'mtime': 1}, + 'author': {'mtime': 1, 'value': {}}, 'layer': {'mtime': 1, 'value': ['public']}, 'tags': {'mtime': 1, 'value': []}, 'mtime': {'value': 0, 'mtime': 1}, @@ -471,14 +474,13 @@ class VolumeTest(tests.Test): 'prop': {'url': url, 'mtime': 1}, }, }, - {'document': 'user'}, {'commit': [[1, 2]]}, ], - [i for i in diff(volume, util.Sequence([[1, None]]))]) + [i for i in diff(volume, toolkit.Sequence([[1, None]]))]) - patch = diff(volume, util.Sequence([[1, None]]), fetch_blobs=True) + patch = diff(volume, toolkit.Sequence([[1, None]]), fetch_blobs=True) self.assertEqual( - {'document': 'document'}, + {'resource': 'document'}, next(patch)) record = next(patch) self.assertEqual(blob, ''.join([i for i in record.pop('blob')])) @@ -488,7 +490,7 @@ class VolumeTest(tests.Test): self.assertEqual( {'guid': guid, 'diff': { 'guid': {'value': guid, 'mtime': 1}, - 'author': {'value': {'principal': {'order': 0, 'role': 2}}, 'mtime': 1}, + 'author': {'mtime': 1, 'value': {}}, 'layer': {'mtime': 1, 'value': ['public']}, 'tags': {'mtime': 1, 'value': []}, 'mtime': {'value': 0, 'mtime': 1}, @@ -496,35 +498,32 @@ class VolumeTest(tests.Test): }}, next(patch)) self.assertEqual( - {'document': 'user'}, - next(patch)) - self.assertEqual( {'commit': [[1, 2]]}, next(patch)) self.assertRaises(StopIteration, next, patch) def test_diff_SkipBrokenBlobUrls(self): - class Document(Resource): + class Document(db.Resource): @db.blob_property() def prop(self, value): return value - volume = Volume('db', [User, Document]) - cp = NodeCommands('guid', volume) + volume = db.Volume('db', [Document]) + cp = NodeRoutes('guid', volume) - guid1 = call(cp, method='POST', document='document', principal='principal', content={}) - call(cp, method='PUT', document='document', guid=guid1, principal='principal', content={'prop': {'url': 'http://foo/bar'}}) - guid2 = call(cp, method='POST', document='document', principal='principal', content={}) + guid1 = call(cp, method='POST', document='document', content={}) + call(cp, method='PUT', document='document', guid=guid1, content={'prop': {'url': 'http://foo/bar'}}) + guid2 = call(cp, method='POST', document='document', content={}) self.utime('db', 1) self.assertEqual([ - {'document': 'document'}, + {'resource': 'document'}, {'guid': guid1, 'diff': { 'guid': {'value': guid1, 'mtime': 1}, - 'author': {'value': {'principal': {'order': 0, 'role': 2}}, 'mtime': 1}, + 'author': {'mtime': 1, 'value': {}}, 'layer': {'mtime': 1, 'value': ['public']}, 'tags': {'mtime': 1, 'value': []}, 'mtime': {'value': 0, 'mtime': 1}, @@ -535,24 +534,23 @@ class VolumeTest(tests.Test): {'guid': guid2, 'diff': { 'guid': {'value': guid2, 'mtime': 1}, - 'author': {'value': {'principal': {'order': 0, 'role': 2}}, 'mtime': 1}, + 'author': {'mtime': 1, 'value': {}}, 'layer': {'mtime': 1, 'value': ['public']}, 'tags': {'mtime': 1, 'value': []}, 'mtime': {'value': 0, 'mtime': 1}, 'ctime': {'value': 0, 'mtime': 1}, }, }, - {'document': 'user'}, {'commit': [[1, 3]]}, ], - [i for i in diff(volume, util.Sequence([[1, None]]), fetch_blobs=False)]) + [i for i in diff(volume, toolkit.Sequence([[1, None]]), fetch_blobs=False)]) self.assertEqual([ - {'document': 'document'}, + {'resource': 'document'}, {'guid': guid1, 'diff': { 'guid': {'value': guid1, 'mtime': 1}, - 'author': {'value': {'principal': {'order': 0, 'role': 2}}, 'mtime': 1}, + 'author': {'mtime': 1, 'value': {}}, 'layer': {'mtime': 1, 'value': ['public']}, 'tags': {'mtime': 1, 'value': []}, 'mtime': {'value': 0, 'mtime': 1}, @@ -562,30 +560,29 @@ class VolumeTest(tests.Test): {'guid': guid2, 'diff': { 'guid': {'value': guid2, 'mtime': 1}, - 'author': {'value': {'principal': {'order': 0, 'role': 2}}, 'mtime': 1}, + 'author': {'mtime': 1, 'value': {}}, 'layer': {'mtime': 1, 'value': ['public']}, 'tags': {'mtime': 1, 'value': []}, 'mtime': {'value': 0, 'mtime': 1}, 'ctime': {'value': 0, 'mtime': 1}, }, }, - {'document': 'user'}, {'commit': [[1, 3]]}, ], - [i for i in diff(volume, util.Sequence([[1, None]]), fetch_blobs=True)]) + [i for i in diff(volume, toolkit.Sequence([[1, None]]), fetch_blobs=True)]) def test_merge_Blobs(self): - class Document(db.Document): + class Document(db.Resource): @db.blob_property() def prop(self, value): return value - volume = Volume('db', [Document]) + volume = db.Volume('db', [Document]) merge(volume, [ - {'document': 'document'}, + {'resource': 'document'}, {'guid': '1', 'diff': { 'guid': {'value': '1', 'mtime': 1.0}, 'ctime': {'value': 2, 'mtime': 2.0}, @@ -611,16 +608,16 @@ class VolumeTest(tests.Test): def test_diff_ByLayers(self): - class Context(Resource): + class Context(db.Resource): pass - class Implementation(Resource): + class Implementation(db.Resource): pass - class Review(Resource): + class Review(db.Resource): pass - volume = Volume('db', [Context, Implementation, Review]) + volume = db.Volume('db', [Context, Implementation, Review]) volume['context'].create({'guid': '0', 'ctime': 1, 'mtime': 1, 'layer': ['layer0', 'common']}) volume['context'].create({'guid': '1', 'ctime': 1, 'mtime': 1, 'layer': 'layer1'}) volume['implementation'].create({'guid': '2', 'ctime': 2, 'mtime': 2, 'layer': 'layer2'}) @@ -633,56 +630,65 @@ class VolumeTest(tests.Test): self.utime('db', 0) self.assertEqual(sorted([ - {'document': 'context'}, + {'resource': 'context'}, {'guid': '0', 'diff': {'tags': {'value': '0', 'mtime': 0}}}, {'guid': '1', 'diff': {'tags': {'value': '1', 'mtime': 0}}}, - {'document': 'implementation'}, + {'resource': 'implementation'}, {'guid': '2', 'diff': {'tags': {'value': '2', 'mtime': 0}}}, - {'document': 'review'}, + {'resource': 'review'}, {'guid': '3', 'diff': {'tags': {'value': '3', 'mtime': 0}}}, {'commit': [[5, 8]]}, ]), - sorted([i for i in diff(volume, util.Sequence([[5, None]]))])) + sorted([i for i in diff(volume, toolkit.Sequence([[5, None]]))])) self.assertEqual(sorted([ - {'document': 'context'}, + {'resource': 'context'}, {'guid': '0', 'diff': {'tags': {'value': '0', 'mtime': 0}}}, {'guid': '1', 'diff': {'tags': {'value': '1', 'mtime': 0}}}, - {'document': 'implementation'}, - {'document': 'review'}, + {'resource': 'implementation'}, + {'resource': 'review'}, {'guid': '3', 'diff': {'tags': {'value': '3', 'mtime': 0}}}, {'commit': [[5, 8]]}, ]), - sorted([i for i in diff(volume, util.Sequence([[5, None]]), layer='layer1')])) + sorted([i for i in diff(volume, toolkit.Sequence([[5, None]]), layer='layer1')])) self.assertEqual(sorted([ - {'document': 'context'}, + {'resource': 'context'}, {'guid': '0', 'diff': {'tags': {'value': '0', 'mtime': 0}}}, - {'document': 'implementation'}, + {'resource': 'implementation'}, {'guid': '2', 'diff': {'tags': {'value': '2', 'mtime': 0}}}, - {'document': 'review'}, + {'resource': 'review'}, {'guid': '3', 'diff': {'tags': {'value': '3', 'mtime': 0}}}, {'commit': [[5, 8]]}, ]), - sorted([i for i in diff(volume, util.Sequence([[5, None]]), layer='layer2')])) + sorted([i for i in diff(volume, toolkit.Sequence([[5, None]]), layer='layer2')])) self.assertEqual(sorted([ - {'document': 'context'}, + {'resource': 'context'}, {'guid': '0', 'diff': {'tags': {'value': '0', 'mtime': 0}}}, - {'document': 'implementation'}, - {'document': 'review'}, + {'resource': 'implementation'}, + {'resource': 'review'}, {'guid': '3', 'diff': {'tags': {'value': '3', 'mtime': 0}}}, {'commit': [[5, 8]]}, ]), - sorted([i for i in diff(volume, util.Sequence([[5, None]]), layer='foo')])) - - -def call(cp, principal=None, content=None, **kwargs): - request = db.Request(**kwargs) - request.principal = principal + sorted([i for i in diff(volume, toolkit.Sequence([[5, None]]), layer='foo')])) + + +def call(routes, method, document=None, guid=None, prop=None, cmd=None, content=None, **kwargs): + path = [] + if document: + path.append(document) + if guid: + path.append(guid) + if prop: + path.append(prop) + request = Request(method=method, path=path) + request.update(kwargs) + request.cmd = cmd request.content = content request.environ = {'HTTP_HOST': '127.0.0.1'} - return cp.call(request, db.Response()) + router = Router(routes) + return router.call(request, Response()) if __name__ == '__main__': diff --git a/tests/units/resources/volume.py b/tests/units/resources/volume.py deleted file mode 100755 index 687f9c8..0000000 --- a/tests/units/resources/volume.py +++ /dev/null @@ -1,444 +0,0 @@ -#!/usr/bin/env python -# sugar-lint: disable - -import os -import json -import time -from os.path import exists - -from __init__ import tests - -from sugar_network import db, node -from sugar_network.resources.volume import Volume, Resource, Commands -from sugar_network.resources.user import User -from sugar_network.toolkit import coroutine, util - - -class VolumeTest(tests.Test): - - def test_Subscribe(self): - - class Document(Resource): - - @db.indexed_property(slot=1) - def prop(self, value): - return value - - volume = Volume('db', [Document]) - cp = TestCommands(volume) - events = [] - - def read_events(): - for event in cp.subscribe(event='!commit'): - if not event.strip(): - continue - assert event.startswith('data: ') - assert event.endswith('\n\n') - event = json.loads(event[6:]) - events.append(event) - - job = coroutine.spawn(read_events) - coroutine.dispatch() - volume['document'].create({'guid': 'guid', 'prop': 'value1'}) - coroutine.dispatch() - volume['document'].update('guid', {'prop': 'value2'}) - coroutine.dispatch() - volume['document'].delete('guid') - coroutine.dispatch() - volume['document'].commit() - coroutine.sleep(.5) - job.kill() - - self.assertEqual([ - {'guid': 'guid', 'document': 'document', 'event': 'create'}, - {'guid': 'guid', 'document': 'document', 'event': 'update'}, - {'guid': 'guid', 'event': 'delete', 'document': u'document'}, - ], - events) - - def test_SubscribeWithPong(self): - volume = Volume('db', []) - cp = TestCommands(volume) - - for event in cp.subscribe(ping=True): - break - self.assertEqual('data: {"event": "pong"}\n\n', event) - - def __test_SubscribeCondition(self): - - class Document(Resource): - - @db.indexed_property(slot=1) - def prop(self, value): - return value - - volume = Volume('db', [Document]) - cp = TestCommands(volume) - events = [] - - def read_events(): - for event in cp.subscribe(db.Request(), db.Response(), only_commits=True): - if not event.strip(): - continue - assert event.startswith('data: ') - assert event.endswith('\n\n') - event = json.loads(event[6:]) - events.append(event) - - job = coroutine.spawn(read_events) - coroutine.dispatch() - volume['document'].create({'guid': 'guid', 'prop': 'value1'}) - coroutine.dispatch() - volume['document'].update('guid', {'prop': 'value2'}) - coroutine.dispatch() - volume['document'].delete('guid') - coroutine.dispatch() - volume['document'].commit() - coroutine.sleep(.5) - job.kill() - - self.assertEqual([ - {'document': 'document', 'event': 'commit'}, - {'document': 'document', 'event': 'commit'}, - {'document': 'document', 'event': 'commit'}, - ], - events) - - def test_Populate(self): - self.touch( - ('db/context/1/1/guid', json.dumps({"value": "1"})), - ('db/context/1/1/ctime', json.dumps({"value": 1})), - ('db/context/1/1/mtime', json.dumps({"value": 1})), - ('db/context/1/1/seqno', json.dumps({"value": 0})), - ('db/context/1/1/type', json.dumps({"value": "activity"})), - ('db/context/1/1/title', json.dumps({"value": {}})), - ('db/context/1/1/summary', json.dumps({"value": {}})), - ('db/context/1/1/description', json.dumps({"value": {}})), - ) - - volume = Volume('db') - cp = TestCommands(volume) - assert exists('db/context/index') - - def test_DefaultAuthor(self): - - class Document(Resource): - pass - - volume = Volume('db', [User, Document]) - cp = TestCommands(volume) - - guid = call(cp, method='POST', document='document', content={}, principal='user') - self.assertEqual( - [{'name': 'user', 'role': 2}], - call(cp, method='GET', document='document', guid=guid, prop='author')) - self.assertEqual( - {'user': {'role': 2, 'order': 0}}, - volume['document'].get(guid)['author']) - - volume['user'].create({'guid': 'user', 'color': '', 'pubkey': '', 'name': 'User'}) - - guid = call(cp, method='POST', document='document', content={}, principal='user') - self.assertEqual( - [{'guid': 'user', 'name': 'User', 'role': 3}], - call(cp, method='GET', document='document', guid=guid, prop='author')) - self.assertEqual( - {'user': {'name': 'User', 'role': 3, 'order': 0}}, - volume['document'].get(guid)['author']) - - def test_FindByAuthor(self): - - class Document(Resource): - pass - - volume = Volume('db', [User, Document]) - cp = TestCommands(volume) - - volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'UserName1'}) - volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User Name2'}) - volume['user'].create({'guid': 'user3', 'color': '', 'pubkey': '', 'name': 'User Name 3'}) - - guid1 = call(cp, method='POST', document='document', content={}, principal='user1') - guid2 = call(cp, method='POST', document='document', content={}, principal='user2') - guid3 = call(cp, method='POST', document='document', content={}, principal='user3') - - self.assertEqual(sorted([ - {'guid': guid1}, - ]), - call(cp, method='GET', document='document', author='UserName1')['result']) - - self.assertEqual(sorted([ - {'guid': guid1}, - ]), - sorted(call(cp, method='GET', document='document', query='author:UserName')['result'])) - self.assertEqual(sorted([ - {'guid': guid1}, - {'guid': guid2}, - {'guid': guid3}, - ]), - sorted(call(cp, method='GET', document='document', query='author:User')['result'])) - self.assertEqual(sorted([ - {'guid': guid2}, - {'guid': guid3}, - ]), - sorted(call(cp, method='GET', document='document', query='author:Name')['result'])) - - def test_PreserveAuthorsOrder(self): - - class Document(Resource): - pass - - volume = Volume('db', [User, Document]) - cp = TestCommands(volume) - - volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'}) - volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User2'}) - volume['user'].create({'guid': 'user3', 'color': '', 'pubkey': '', 'name': 'User3'}) - - guid = call(cp, method='POST', document='document', content={}, principal='user1') - call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='user2', role=0) - call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='user3', role=0) - - self.assertEqual([ - {'guid': 'user1', 'name': 'User1', 'role': 3}, - {'guid': 'user2', 'name': 'User2', 'role': 1}, - {'guid': 'user3', 'name': 'User3', 'role': 1}, - ], - call(cp, method='GET', document='document', guid=guid, prop='author')) - self.assertEqual({ - 'user1': {'name': 'User1', 'role': 3, 'order': 0}, - 'user2': {'name': 'User2', 'role': 1, 'order': 1}, - 'user3': {'name': 'User3', 'role': 1, 'order': 2}, - }, - volume['document'].get(guid)['author']) - - call(cp, method='PUT', document='document', guid=guid, cmd='userdel', user='user2', principal='user1') - call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='user2', role=0) - - self.assertEqual([ - {'guid': 'user1', 'name': 'User1', 'role': 3}, - {'guid': 'user3', 'name': 'User3', 'role': 1}, - {'guid': 'user2', 'name': 'User2', 'role': 1}, - ], - call(cp, method='GET', document='document', guid=guid, prop='author')) - self.assertEqual({ - 'user1': {'name': 'User1', 'role': 3, 'order': 0}, - 'user3': {'name': 'User3', 'role': 1, 'order': 2}, - 'user2': {'name': 'User2', 'role': 1, 'order': 3}, - }, - volume['document'].get(guid)['author']) - - call(cp, method='PUT', document='document', guid=guid, cmd='userdel', user='user2', principal='user1') - call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='user2', role=0) - - self.assertEqual([ - {'guid': 'user1', 'name': 'User1', 'role': 3}, - {'guid': 'user3', 'name': 'User3', 'role': 1}, - {'guid': 'user2', 'name': 'User2', 'role': 1}, - ], - call(cp, method='GET', document='document', guid=guid, prop='author')) - self.assertEqual({ - 'user1': {'name': 'User1', 'role': 3, 'order': 0}, - 'user3': {'name': 'User3', 'role': 1, 'order': 2}, - 'user2': {'name': 'User2', 'role': 1, 'order': 3}, - }, - volume['document'].get(guid)['author']) - - call(cp, method='PUT', document='document', guid=guid, cmd='userdel', user='user3', principal='user1') - call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='user3', role=0) - - self.assertEqual([ - {'guid': 'user1', 'name': 'User1', 'role': 3}, - {'guid': 'user2', 'name': 'User2', 'role': 1}, - {'guid': 'user3', 'name': 'User3', 'role': 1}, - ], - call(cp, method='GET', document='document', guid=guid, prop='author')) - self.assertEqual({ - 'user1': {'name': 'User1', 'role': 3, 'order': 0}, - 'user2': {'name': 'User2', 'role': 1, 'order': 3}, - 'user3': {'name': 'User3', 'role': 1, 'order': 4}, - }, - volume['document'].get(guid)['author']) - - def test_AddUser(self): - - class Document(Resource): - pass - - volume = Volume('db', [User, Document]) - cp = TestCommands(volume) - - volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'}) - volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User2'}) - - guid = call(cp, method='POST', document='document', content={}, principal='user1') - self.assertEqual([ - {'guid': 'user1', 'name': 'User1', 'role': 3}, - ], - call(cp, method='GET', document='document', guid=guid, prop='author')) - self.assertEqual({ - 'user1': {'name': 'User1', 'role': 3, 'order': 0}, - }, - volume['document'].get(guid)['author']) - - call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='user2', role=2) - self.assertEqual([ - {'guid': 'user1', 'name': 'User1', 'role': 3}, - {'guid': 'user2', 'name': 'User2', 'role': 3}, - ], - call(cp, method='GET', document='document', guid=guid, prop='author')) - self.assertEqual({ - 'user1': {'name': 'User1', 'role': 3, 'order': 0}, - 'user2': {'name': 'User2', 'role': 3, 'order': 1}, - }, - volume['document'].get(guid)['author']) - - call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='User3', role=3) - self.assertEqual([ - {'guid': 'user1', 'name': 'User1', 'role': 3}, - {'guid': 'user2', 'name': 'User2', 'role': 3}, - {'name': 'User3', 'role': 2}, - ], - call(cp, method='GET', document='document', guid=guid, prop='author')) - self.assertEqual({ - 'user1': {'name': 'User1', 'role': 3, 'order': 0}, - 'user2': {'name': 'User2', 'role': 3, 'order': 1}, - 'User3': {'role': 2, 'order': 2}, - }, - volume['document'].get(guid)['author']) - - call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='User4', role=4) - self.assertEqual([ - {'guid': 'user1', 'name': 'User1', 'role': 3}, - {'guid': 'user2', 'name': 'User2', 'role': 3}, - {'name': 'User3', 'role': 2}, - {'name': 'User4', 'role': 0}, - ], - call(cp, method='GET', document='document', guid=guid, prop='author')) - self.assertEqual({ - 'user1': {'name': 'User1', 'role': 3, 'order': 0}, - 'user2': {'name': 'User2', 'role': 3, 'order': 1}, - 'User3': {'role': 2, 'order': 2}, - 'User4': {'role': 0, 'order': 3}, - }, - volume['document'].get(guid)['author']) - - def test_UpdateAuthor(self): - - class Document(Resource): - pass - - volume = Volume('db', [User, Document]) - cp = TestCommands(volume) - - volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'}) - guid = call(cp, method='POST', document='document', content={}, principal='user1') - - call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='User2', role=0) - self.assertEqual([ - {'guid': 'user1', 'name': 'User1', 'role': 3}, - {'name': 'User2', 'role': 0}, - ], - call(cp, method='GET', document='document', guid=guid, prop='author')) - self.assertEqual({ - 'user1': {'name': 'User1', 'role': 3, 'order': 0}, - 'User2': {'role': 0, 'order': 1}, - }, - volume['document'].get(guid)['author']) - - call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='user1', role=0) - self.assertEqual([ - {'guid': 'user1', 'name': 'User1', 'role': 1}, - {'name': 'User2', 'role': 0}, - ], - call(cp, method='GET', document='document', guid=guid, prop='author')) - self.assertEqual({ - 'user1': {'name': 'User1', 'role': 1, 'order': 0}, - 'User2': {'role': 0, 'order': 1}, - }, - volume['document'].get(guid)['author']) - - call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='User2', role=2) - self.assertEqual([ - {'guid': 'user1', 'name': 'User1', 'role': 1}, - {'name': 'User2', 'role': 2}, - ], - call(cp, method='GET', document='document', guid=guid, prop='author')) - self.assertEqual({ - 'user1': {'name': 'User1', 'role': 1, 'order': 0}, - 'User2': {'role': 2, 'order': 1}, - }, - volume['document'].get(guid)['author']) - - def test_DelUser(self): - - class Document(Resource): - pass - - volume = Volume('db', [User, Document]) - cp = TestCommands(volume) - - volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'}) - volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User2'}) - guid = call(cp, method='POST', document='document', content={}, principal='user1') - call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='user2') - call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='User3') - self.assertEqual([ - {'guid': 'user1', 'name': 'User1', 'role': 3}, - {'guid': 'user2', 'name': 'User2', 'role': 1}, - {'name': 'User3', 'role': 0}, - ], - call(cp, method='GET', document='document', guid=guid, prop='author')) - self.assertEqual({ - 'user1': {'name': 'User1', 'role': 3, 'order': 0}, - 'user2': {'name': 'User2', 'role': 1, 'order': 1}, - 'User3': {'role': 0, 'order': 2}, - }, - volume['document'].get(guid)['author']) - - # Do not remove yourself - self.assertRaises(RuntimeError, call, cp, method='PUT', document='document', guid=guid, cmd='userdel', user='user1', principal='user1') - self.assertRaises(RuntimeError, call, cp, method='PUT', document='document', guid=guid, cmd='userdel', user='user2', principal='user2') - - call(cp, method='PUT', document='document', guid=guid, cmd='userdel', user='user1', principal='user2') - self.assertEqual([ - {'guid': 'user2', 'name': 'User2', 'role': 1}, - {'name': 'User3', 'role': 0}, - ], - call(cp, method='GET', document='document', guid=guid, prop='author')) - self.assertEqual({ - 'user2': {'name': 'User2', 'role': 1, 'order': 1}, - 'User3': {'role': 0, 'order': 2}, - }, - volume['document'].get(guid)['author']) - - call(cp, method='PUT', document='document', guid=guid, cmd='userdel', user='User3', principal='user2') - self.assertEqual([ - {'guid': 'user2', 'name': 'User2', 'role': 1}, - ], - call(cp, method='GET', document='document', guid=guid, prop='author')) - self.assertEqual({ - 'user2': {'name': 'User2', 'role': 1, 'order': 1}, - }, - volume['document'].get(guid)['author']) - - -class TestCommands(db.VolumeCommands, Commands): - - def __init__(self, volume): - db.VolumeCommands.__init__(self, volume) - Commands.__init__(self) - self.volume.connect(self.broadcast) - - -def call(cp, principal=None, content=None, **kwargs): - request = db.Request(**kwargs) - request.principal = principal - request.content = content - request.environ = {'HTTP_HOST': '127.0.0.1'} - request.commands = cp - return cp.call(request, db.Response()) - - -if __name__ == '__main__': - tests.main() diff --git a/tests/units/toolkit/__main__.py b/tests/units/toolkit/__main__.py index 9871e6b..841711e 100644 --- a/tests/units/toolkit/__main__.py +++ b/tests/units/toolkit/__main__.py @@ -6,9 +6,10 @@ from http import * from lsb_release import * from mountpoints import * from rrd import * -from util import * +from toolkit import * from options import * from spec import * +from router import * if __name__ == '__main__': tests.main() diff --git a/tests/units/toolkit/http.py b/tests/units/toolkit/http.py index f1fe10e..4117cbc 100755 --- a/tests/units/toolkit/http.py +++ b/tests/units/toolkit/http.py @@ -6,8 +6,8 @@ import select from __init__ import tests -from sugar_network import db, client as local -from sugar_network.db import router +from sugar_network import client as local +from sugar_network.toolkit.router import route, Router, Request from sugar_network.toolkit import coroutine, http @@ -15,18 +15,17 @@ class HTTPTest(tests.Test): def test_Subscribe(self): - class CommandsProcessor(db.CommandsProcessor): + class CommandsProcessor(object): events = [] - @router.route('GET', '/') + @route('GET', cmd='subscribe') def subscribe(self, request, response): - assert request.get('cmd') == 'subscribe' while CommandsProcessor.events: - coroutine.sleep(.3) + coroutine.sleep(.1) yield CommandsProcessor.events.pop(0) + '\n' - self.server = coroutine.WSGIServer(('127.0.0.1', local.ipc_port.value), router.Router(CommandsProcessor())) + self.server = coroutine.WSGIServer(('127.0.0.1', local.ipc_port.value), Router(CommandsProcessor())) coroutine.spawn(self.server.serve_forever) coroutine.dispatch() client = http.Client('http://127.0.0.1:%s' % local.ipc_port.value) @@ -57,29 +56,33 @@ class HTTPTest(tests.Test): def test_call_ReturnStream(self): - class Commands(db.CommandsProcessor): + class Commands(object): - @db.volume_command(method='GET', cmd='f1', mime_type='application/json') + @route('GET', cmd='f1', mime_type='application/json') def f1(self): yield json.dumps('result') - @db.volume_command(method='GET', cmd='f2', mime_type='foo/bar') + @route('GET', cmd='f2', mime_type='foo/bar') def f2(self): yield json.dumps('result') - self.server = coroutine.WSGIServer(('127.0.0.1', local.ipc_port.value), router.Router(Commands())) + self.server = coroutine.WSGIServer(('127.0.0.1', local.ipc_port.value), Router(Commands())) coroutine.spawn(self.server.serve_forever) coroutine.dispatch() client = http.Client('http://127.0.0.1:%s' % local.ipc_port.value) - request = db.Request() - request['method'] = 'GET' - request['cmd'] = 'f1' + request = Request({ + 'REQUEST_METHOD': 'GET', + 'PATH_INFO': '/', + 'QUERY_STRING': 'cmd=f1', + }) self.assertEqual('result', client.call(request)) - request = db.Request() - request['method'] = 'GET' - request['cmd'] = 'f2' + request = Request({ + 'REQUEST_METHOD': 'GET', + 'PATH_INFO': '/', + 'QUERY_STRING': 'cmd=f2', + }) self.assertEqual('result', json.load(client.call(request))) diff --git a/tests/units/toolkit/router.py b/tests/units/toolkit/router.py new file mode 100755 index 0000000..32e26f3 --- /dev/null +++ b/tests/units/toolkit/router.py @@ -0,0 +1,1257 @@ +#!/usr/bin/env python +# sugar-lint: disable + +import os +import json +from email.utils import formatdate +from cStringIO import StringIO + +from __init__ import tests, src_root + +from sugar_network import db +from sugar_network.toolkit.router import Blob, Router, Request, _parse_accept_language, route, fallbackroute, preroute, postroute, _filename +from sugar_network.toolkit import default_lang, http + + +class RouterTest(tests.Test): + + def test_routes_Exact(self): + + class Routes(object): + + @route('PROBE') + def command_1(self): + return 'command_1' + + @route('PROBE', cmd='command_2') + def command_2(self): + return 'command_2' + + @route('PROBE', ['resource']) + def command_3(self): + return 'command_3' + + @route('PROBE', ['resource'], cmd='command_4') + def command_4(self): + return 'command_4' + + @route('PROBE', ['resource', 'guid']) + def command_5(self): + return 'command_5' + + @route('PROBE', ['resource', 'guid'], cmd='command_6') + def command_6(self): + return 'command_6' + + @route('PROBE', ['resource', 'guid', 'prop']) + def command_7(self): + return 'command_7' + + @route('PROBE', ['resource', 'guid', 'prop'], cmd='command_8') + def command_8(self): + return 'command_8' + + router = Router(Routes()) + status = [] + + self.assertEqual( + ['command_1'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': '', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_1')))]), status[-1]) + + self.assertEqual( + ['command_2'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': 'cmd=command_2', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_2')))]), status[-1]) + + self.assertEqual( + ['command_3'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/resource', + 'QUERY_STRING': '', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_3')))]), status[-1]) + + self.assertEqual( + ['command_4'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/resource', + 'QUERY_STRING': 'cmd=command_4', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_4')))]), status[-1]) + + self.assertEqual( + ['command_5'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/resource/guid', + 'QUERY_STRING': '', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_5')))]), status[-1]) + + self.assertEqual( + ['command_6'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/resource/guid', + 'QUERY_STRING': 'cmd=command_6', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_6')))]), status[-1]) + + self.assertEqual( + ['command_7'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/resource/guid/prop', + 'QUERY_STRING': '', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_7')))]), status[-1]) + + self.assertEqual( + ['command_8'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/resource/guid/prop', + 'QUERY_STRING': 'cmd=command_8', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_8')))]), status[-1]) + + self.assertEqual( + ['{"request": "/*/*/*", "error": "Path not found"}'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/*/*/*'}, lambda *args: None)]) + self.assertEqual( + ['{"request": "/*", "error": "Path not found"}'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/*'}, lambda *args: None)]) + self.assertEqual( + ['{"request": "/?cmd=*", "error": "No such operation"}'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/', 'QUERY_STRING': 'cmd=*'}, lambda *args: None)]) + self.assertEqual( + ['{"request": "/", "error": "No such operation"}'], + [i for i in router({'REQUEST_METHOD': 'GET', 'PATH_INFO': '/'}, lambda *args: None)]) + + def test_routes_TailWildcards(self): + + class Routes(object): + + @route('PROBE', ['resource', 'guid', None]) + def command_1(self): + return 'command_1' + + @route('PROBE', ['resource', 'guid', None], cmd='command_2') + def command_2(self): + return 'command_2' + + @route('PROBE', ['resource', None, None]) + def command_3(self): + return 'command_3' + + @route('PROBE', ['resource', None, None], cmd='command_4') + def command_4(self): + return 'command_4' + + @route('PROBE', [None, None, None]) + def command_5(self): + return 'command_5' + + @route('PROBE', [None, None, None], cmd='command_6') + def command_6(self): + return 'command_6' + + router = Router(Routes()) + status = [] + + self.assertEqual( + ['command_1'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/resource/guid/*', + 'QUERY_STRING': '', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_1')))]), status[-1]) + + self.assertEqual( + ['command_2'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/resource/guid/*', + 'QUERY_STRING': 'cmd=command_2', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_2')))]), status[-1]) + + self.assertEqual( + ['command_3'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/resource/guid2/prop', + 'QUERY_STRING': '', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_3')))]), status[-1]) + + self.assertEqual( + ['command_4'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/resource/guid2/prop', + 'QUERY_STRING': 'cmd=command_4', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_4')))]), status[-1]) + + self.assertEqual( + ['command_5'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/*/guid/prop', + 'QUERY_STRING': '', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_5')))]), status[-1]) + + self.assertEqual( + ['command_6'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/*/guid/prop', + 'QUERY_STRING': 'cmd=command_6', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_6')))]), status[-1]) + + self.assertEqual( + ['{"request": "/", "error": "Path not found"}'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/'}, lambda *args: None)]) + self.assertEqual( + ['{"request": "/*/*/*?cmd=*", "error": "No such operation"}'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/*/*/*', 'QUERY_STRING': 'cmd=*'}, lambda *args: None)]) + self.assertEqual( + ['{"request": "/*/*/*", "error": "No such operation"}'], + [i for i in router({'REQUEST_METHOD': 'GET', 'PATH_INFO': '/*/*/*'}, lambda *args: None)]) + + def test_routes_FreeWildcards(self): + + class Routes(object): + + @route('PROBE', ['resource1', None, 'prop1']) + def command_1(self): + return 'command_1' + + @route('PROBE', ['resource1', None, 'prop1'], cmd='command_2') + def command_2(self): + return 'command_2' + + @route('PROBE', [None, None, 'prop2']) + def command_3(self): + return 'command_3' + + @route('PROBE', [None, None, 'prop2'], cmd='command_4') + def command_4(self): + return 'command_4' + + router = Router(Routes()) + status = [] + + self.assertEqual( + ['command_1'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/resource1/*/prop1', + 'QUERY_STRING': '', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_1')))]), status[-1]) + + self.assertEqual( + ['command_2'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/resource1/*/prop1', + 'QUERY_STRING': 'cmd=command_2', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_2')))]), status[-1]) + + self.assertEqual( + ['command_3'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/*/*/prop2', + 'QUERY_STRING': '', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_3')))]), status[-1]) + + self.assertEqual( + ['command_4'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/*/*/prop2', + 'QUERY_STRING': 'cmd=command_4', + }, lambda *args: status.append(args))]) + self.assertEqual(('200 OK', [('content-length', str(len('command_4')))]), status[-1]) + + self.assertEqual( + ['{"request": "/*/*/prop3", "error": "Path not found"}'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/*/*/prop3'}, lambda *args: None)]) + self.assertEqual( + ['{"request": "/*/*/prop2", "error": "No such operation"}'], + [i for i in router({'REQUEST_METHOD': 'GET', 'PATH_INFO': '/*/*/prop2'}, lambda *args: None)]) + + self.assertEqual( + ['{"request": "/", "error": "Path not found"}'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/'}, lambda *args: None)]) + self.assertEqual( + ['{"request": "/*/*/*?cmd=*", "error": "Path not found"}'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/*/*/*', 'QUERY_STRING': 'cmd=*'}, lambda *args: None)]) + self.assertEqual( + ['{"request": "/*/*/prop2", "error": "No such operation"}'], + [i for i in router({'REQUEST_METHOD': 'GET', 'PATH_INFO': '/*/*/prop2'}, lambda *args: None)]) + + def test_routes_Fallback(self): + + class Routes(object): + + @fallbackroute() + def fallback(self): + return 'fallback' + + @fallbackroute('PROBE2') + def fallback2(self): + return 'fallback2' + + @route('PROBE', ['exists']) + def exists(self): + return 'exists' + + router = Router(Routes()) + status = [] + + self.assertEqual( + ['exists'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/exists'}, lambda *args: None)]) + self.assertEqual( + ['fallback'], + [i for i in router({'REQUEST_METHOD': 'PUT', 'PATH_INFO': '/'}, lambda *args: None)]) + self.assertEqual( + ['fallback'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/*'}, lambda *args: None)]) + self.assertEqual( + ['fallback'], + [i for i in router({'REQUEST_METHOD': 'GET', 'PATH_INFO': '/*/*/*'}, lambda *args: None)]) + + self.assertEqual( + ['fallback2'], + [i for i in router({'REQUEST_METHOD': 'PROBE2', 'PATH_INFO': '/'}, lambda *args: None)]) + self.assertEqual( + ['fallback2'], + [i for i in router({'REQUEST_METHOD': 'PROBE2', 'PATH_INFO': '/*/*/*'}, lambda *args: None)]) + self.assertEqual( + ['fallback'], + [i for i in router({'REQUEST_METHOD': 'PROBE3', 'PATH_INFO': '/*/*/*/*/*'}, lambda *args: None)]) + + def test_routes_FallbackForCommands(self): + + class Routes(object): + + @fallbackroute() + def fallback(self): + return 'fallback' + + @fallbackroute('PROBE1', ['raise', 'fail']) + def fallback2(self): + return 'fallback2' + + @route('PROBE2', [None, None]) + def exists(self): + return 'exists' + + @route('PROBE3', [None, None], cmd='CMD') + def exists2(self): + return 'exists2' + + router = Router(Routes()) + + self.assertEqual( + ['fallback'], + [i for i in router({'REQUEST_METHOD': 'PROBE3', 'PATH_INFO': '/raise/fail', 'QUERY_STRING': 'cmd=FOO'}, lambda *args: None)]) + + def test_routes_FallbackAndRegularRouteOnTheSameLevel(self): + + class Routes(object): + + @fallbackroute() + def fallback(self): + return 'fallback' + + @route('PROBE') + def exists(self): + return 'exists' + + router = Router(Routes()) + status = [] + + self.assertEqual( + ['exists'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/'}, lambda *args: None)]) + self.assertEqual( + ['fallback'], + [i for i in router({'REQUEST_METHOD': 'GET', 'PATH_INFO': '/'}, lambda *args: None)]) + + def test_routes_CheckFallbacksBeforeWildecards(self): + + class Routes(object): + + @fallbackroute('PROBE', ['static']) + def fallback(self): + return 'fallback' + + @route('PROBE', [None]) + def wildcards(self): + return 'wildcards' + + router = Router(Routes()) + status = [] + + self.assertEqual( + ['fallback'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/static'}, lambda *args: None)]) + self.assertEqual( + ['wildcards'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/foo'}, lambda *args: None)]) + + def test_routes_FallbackForTailedPaths(self): + + class Routes(object): + + @fallbackroute('PROBE', ['static']) + def fallback(self, request): + return '/'.join(request.path) + + router = Router(Routes()) + status = [] + + self.assertEqual( + ['static'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/static'}, lambda *args: None)]) + self.assertEqual( + ['static/foo'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/static/foo'}, lambda *args: None)]) + self.assertEqual( + ['static/foo/bar'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/static/foo/bar'}, lambda *args: None)]) + + def test_routes_ParentClasses(self): + calls = [] + + class Parent(object): + + @route('PROBE') + def probe(self): + return 'probe' + + class Child(Parent): + pass + + router = Router(Child()) + + self.assertEqual( + ['probe'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/'}, lambda *args: None)]) + + def test_routes_OverrideInChildClass(self): + calls = [] + + class Parent(object): + + @route('PROBE') + def probe(self): + return 'probe-1' + + @route('COMMON') + def common(self): + return 'common' + + class Child(Parent): + + @route('PROBE') + def probe(self): + return 'probe-2' + + @route('PARTICULAR') + def particular(self): + return 'particular' + + router = Router(Child()) + + self.assertEqual( + ['probe-2'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/'}, lambda *args: None)]) + self.assertEqual( + ['common'], + [i for i in router({'REQUEST_METHOD': 'COMMON', 'PATH_INFO': '/'}, lambda *args: None)]) + self.assertEqual( + ['particular'], + [i for i in router({'REQUEST_METHOD': 'PARTICULAR', 'PATH_INFO': '/'}, lambda *args: None)]) + + def test_routes_Pre(self): + + class Routes(object): + + @route('PROBE') + def ok(self, request, response): + return request['probe'] + + @preroute + def preroute(self, op, request): + request['probe'] = 'request' + + router = Router(Routes()) + + self.assertEqual( + ['request'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/'}, lambda *args: None)]) + + def test_routes_Post(self): + postroutes = [] + + class Routes(object): + + @route('OK') + def ok(self): + return 'ok' + + @route('FAIL') + def fail(self, request, response): + raise Exception('fail') + + @postroute + def postroute(self, request, response, result, exception): + postroutes.append((result, str(exception))) + + router = Router(Routes()) + + self.assertEqual( + ['ok'], + [i for i in router({'REQUEST_METHOD': 'OK', 'PATH_INFO': '/'}, lambda *args: None)]) + self.assertEqual(('ok', 'None'), postroutes[-1]) + + self.assertEqual( + ['{"request": "/", "error": "fail"}'], + [i for i in router({'REQUEST_METHOD': 'FAIL', 'PATH_INFO': '/'}, lambda *args: None)]) + self.assertEqual((None, 'fail'), postroutes[-1]) + + def test_routes_WildcardsAsLastResort(self): + + class Routes(object): + + @route('PROBE', ['exists']) + def exists(self): + return 'exists' + + @route('PROBE', ['exists', 'deep']) + def exists_deep(self): + return 'exists/deep' + + @route('GET', [None]) + def wildcards(self): + return '*' + + @route('GET', [None, None]) + def wildcards_deep(self): + return '*/*' + + router = Router(Routes()) + status = [] + + self.assertEqual( + ['exists'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/exists'}, lambda *args: None)]) + self.assertEqual( + ['exists/deep'], + [i for i in router({'REQUEST_METHOD': 'PROBE', 'PATH_INFO': '/exists/deep'}, lambda *args: None)]) + self.assertEqual( + ['*'], + [i for i in router({'REQUEST_METHOD': 'GET', 'PATH_INFO': '/exists'}, lambda *args: None)]) + self.assertEqual( + ['*/*'], + [i for i in router({'REQUEST_METHOD': 'GET', 'PATH_INFO': '/exists/deep'}, lambda *args: None)]) + self.assertEqual( + ['*'], + [i for i in router({'REQUEST_METHOD': 'GET', 'PATH_INFO': '/foo'}, lambda *args: None)]) + self.assertEqual( + ['*/*'], + [i for i in router({'REQUEST_METHOD': 'GET', 'PATH_INFO': '/foo/bar'}, lambda *args: None)]) + + def test_Request_Read(self): + + class Stream(object): + + def __init__(self, value): + self.pos = 0 + self.value = value + + def read(self, size): + assert self.pos + size <= len(self.value) + result = self.value[self.pos:self.pos + size] + self.pos += size + return result + + request = Request({'PATH_INFO': '/', 'REQUEST_METHOD': 'GET', 'wsgi.input': Stream('123')}) + request.content_length = len(request.content_stream.value) + self.assertEqual('123', request.read()) + self.assertEqual('', request.read()) + self.assertEqual('', request.read(10)) + + request = Request({'PATH_INFO': '/', 'REQUEST_METHOD': 'GET', 'wsgi.input': Stream('123')}) + request.content_length = len(request.content_stream.value) + self.assertEqual('123', request.read(10)) + + request = Request({'PATH_INFO': '/', 'REQUEST_METHOD': 'GET', 'wsgi.input': Stream('123')}) + request.content_length = len(request.content_stream.value) + self.assertEqual('1', request.read(1)) + self.assertEqual('2', request.read(1)) + self.assertEqual('3', request.read()) + + def test_IntArguments(self): + + class Routes(object): + + @route('PROBE', arguments={'arg': int}) + def probe(self, arg): + return json.dumps(arg) + + router = Router(Routes()) + + self.assertEqual( + ['null'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + }, lambda *args: None)]) + self.assertEqual( + ['-1'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': 'arg=-1', + }, lambda *args: None)]) + self.assertEqual( + ['2'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': 'arg=1&arg=2', + }, lambda *args: None)]) + self.assertEqual( + ['0'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': 'arg=', + }, lambda *args: None)]) + self.assertEqual( + ['null'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + }, lambda *args: None)]) + + def test_BoolArguments(self): + + class Routes(object): + + @route('PROBE', arguments={'arg': bool}) + def probe(self, arg): + return json.dumps(arg) + + router = Router(Routes()) + + self.assertEqual( + ['null'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + }, lambda *args: None)]) + self.assertEqual( + ['true'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': 'arg=1', + }, lambda *args: None)]) + self.assertEqual( + ['true'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': 'arg=on', + }, lambda *args: None)]) + self.assertEqual( + ['true'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': 'arg=true', + }, lambda *args: None)]) + self.assertEqual( + ['false'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': 'arg=foo', + }, lambda *args: None)]) + self.assertEqual( + ['true'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': 'arg=', + }, lambda *args: None)]) + self.assertEqual( + ['true'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': 'arg', + }, lambda *args: None)]) + self.assertEqual( + ['null'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + }, lambda *args: None)]) + + def test_ListArguments(self): + + class Routes(object): + + @route('PROBE', arguments={'arg': list}) + def probe(self, arg): + return json.dumps(arg) + + router = Router(Routes()) + + self.assertEqual( + ['null'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + }, lambda *args: None)]) + self.assertEqual( + ['["a1"]'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': 'arg=a1', + }, lambda *args: None)]) + self.assertEqual( + ['["a1", "a2"]'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': 'arg=a1,a2', + }, lambda *args: None)]) + self.assertEqual( + ['["a1", "a2", "a3"]'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': 'arg=a1&arg=a2&arg=a3', + }, lambda *args: None)]) + self.assertEqual( + ['[]'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': 'arg=', + }, lambda *args: None)]) + self.assertEqual( + ['null'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + }, lambda *args: None)]) + + def test_ArgumentDefaults(self): + + class Routes(object): + + @route('PROBE', arguments={'a1': -1, 'a2': False, 'a3': [None]}, mime_type='application/json') + def probe(self, a1, a2, a3): + return (a1, a2, a3) + + router = Router(Routes()) + + self.assertEqual( + ['[-1, false, [null]]'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + }, lambda *args: None)]) + self.assertEqual( + ['[1, true, ["3", "4"]]'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': 'a1=1&a2=1&a3=3,4', + }, lambda *args: None)]) + self.assertEqual( + ['[0, true, []]'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + 'QUERY_STRING': 'a1=&a2=&a3=', + }, lambda *args: None)]) + self.assertEqual( + ['[-1, false, [null]]'], + [i for i in router({ + 'REQUEST_METHOD': 'PROBE', + 'PATH_INFO': '/', + }, lambda *args: None)]) + + def test_StreamedResponse(self): + + class CommandsProcessor(object): + + @route('GET') + def get_stream(self, response): + return StringIO('stream') + + router = Router(CommandsProcessor()) + + response = router({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'GET', + }, + lambda *args: None) + self.assertEqual('stream', ''.join([i for i in response])) + + def test_EmptyResponse(self): + + class CommandsProcessor(object): + + @route('GET', [], '1', mime_type='application/octet-stream') + def get_binary(self, response): + pass + + @route('GET', [], '2', mime_type='application/json') + def get_json(self, response): + pass + + @route('GET', [], '3') + def no_get(self, response): + pass + + router = Router(CommandsProcessor()) + + response = router({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'GET', + 'QUERY_STRING': 'cmd=1', + }, + lambda *args: None) + self.assertEqual('', ''.join([i for i in response])) + + response = router({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'GET', + 'QUERY_STRING': 'cmd=2', + }, + lambda *args: None) + self.assertEqual('null', ''.join([i for i in response])) + + response = router({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'GET', + 'QUERY_STRING': 'cmd=3', + }, + lambda *args: None) + self.assertEqual('', ''.join([i for i in response])) + + def test_StatusWOResult(self): + + class Status(http.Status): + status = '001 Status' + headers = {'status-header': 'value'} + + class CommandsProcessor(object): + + @route('GET') + def get(self, response): + raise Status('Status-Error') + + router = Router(CommandsProcessor()) + + response = [] + reply = router({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'GET', + }, + lambda status, headers: response.extend([status, dict(headers)])) + error = json.dumps({'request': '/', 'error': 'Status-Error'}) + self.assertEqual(error, ''.join([i for i in reply])) + self.assertEqual([ + '001 Status', + {'content-length': str(len(error)), 'content-type': 'application/json', 'status-header': 'value'}, + ], + response) + + def test_ErrorInHEAD(self): + + class Status(http.Status): + status = '001 Status' + + class CommandsProcessor(object): + + @route('HEAD') + def get(self, response): + raise Status('Status-Error') + + router = Router(CommandsProcessor()) + + response = [] + reply = router({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'HEAD', + }, + lambda status, headers: response.extend([status, dict(headers)])) + self.assertEqual('', ''.join([i for i in reply])) + self.assertEqual([ + '001 Status', + {'X-SN-error': '"Status-Error"'}, + ], + response) + + def test_StatusPass(self): + + class StatusPass(http.StatusPass): + status = '001 StatusPass' + headers = {'statuspass-header': 'value'} + result = 'result' + + class CommandsProcessor(object): + + @route('GET') + def get(self, response): + raise StatusPass('Status-Error') + + router = Router(CommandsProcessor()) + + response = [] + reply = router({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'GET', + }, + lambda status, headers: response.extend([status, dict(headers)])) + error = '' + self.assertEqual(error, ''.join([i for i in reply])) + self.assertEqual([ + '001 StatusPass', + {'content-length': str(len(error)), 'statuspass-header': 'value'}, + ], + response) + + def test_BlobsRedirects(self): + URL = 'http://sugarlabs.org' + + class CommandsProcessor(object): + + @route('GET') + def get(self, response): + return Blob(url=URL) + + router = Router(CommandsProcessor()) + + response = [] + reply = router({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'GET', + }, + lambda status, headers: response.extend([status, dict(headers)])) + error = '' + self.assertEqual(error, ''.join([i for i in reply])) + self.assertEqual([ + '303 See Other', + {'content-length': '0', 'location': URL}, + ], + response) + + def test_LastModified(self): + + class CommandsProcessor(object): + + @route('GET') + def get(self, request, response): + response.last_modified = 10 + return 'ok' + + router = Router(CommandsProcessor()) + + response = [] + reply = router({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'GET', + }, + lambda status, headers: response.extend([status, dict(headers)])) + result = 'ok' + self.assertEqual(result, ''.join([i for i in reply])) + self.assertEqual([ + '200 OK', + {'last-modified': formatdate(10, localtime=False, usegmt=True), 'content-length': str(len(result))}, + ], + response) + + def test_IfModifiedSince(self): + + class CommandsProcessor(object): + + @route('GET') + def get(self, request): + if not request.if_modified_since or request.if_modified_since >= 10: + return 'ok' + else: + raise http.NotModified() + + router = Router(CommandsProcessor()) + + response = [] + reply = router({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'GET', + }, + lambda status, headers: response.extend([status, dict(headers)])) + result = 'ok' + self.assertEqual(result, ''.join([i for i in reply])) + self.assertEqual([ + '200 OK', + {'content-length': str(len(result))}, + ], + response) + + response = [] + reply = router({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'GET', + 'HTTP_IF_MODIFIED_SINCE': formatdate(11, localtime=False, usegmt=True), + }, + lambda status, headers: response.extend([status, dict(headers)])) + result = 'ok' + self.assertEqual(result, ''.join([i for i in reply])) + self.assertEqual([ + '200 OK', + {'content-length': str(len(result))}, + ], + response) + + response = [] + reply = router({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'GET', + 'HTTP_IF_MODIFIED_SINCE': formatdate(9, localtime=False, usegmt=True), + }, + lambda status, headers: response.extend([status, dict(headers)])) + result = '' + self.assertEqual(result, ''.join([i for i in reply])) + self.assertEqual([ + '304 Not Modified', + {'content-length': str(len(result))}, + ], + response) + + def test_Request_MultipleQueryArguments(self): + request = Request({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'GET', + 'QUERY_STRING': 'a1=v1&a2=v2&a1=v3&a3=v4&a1=v5&a3=v6', + }) + self.assertEqual( + {'a1': ['v1', 'v3', 'v5'], 'a2': 'v2', 'a3': ['v4', 'v6']}, + dict(request)) + + def test_Register_UrlPath(self): + self.assertEqual( + [], + Request({'REQUEST_METHOD': 'GET', 'PATH_INFO': ''}).path) + self.assertEqual( + [], + Request({'REQUEST_METHOD': 'GET', 'PATH_INFO': '/'}).path) + self.assertEqual( + ['foo'], + Request({'REQUEST_METHOD': 'GET', 'PATH_INFO': 'foo'}).path) + self.assertEqual( + ['foo', 'bar'], + Request({'REQUEST_METHOD': 'GET', 'PATH_INFO': 'foo/bar'}).path) + self.assertEqual( + ['foo', 'bar'], + Request({'REQUEST_METHOD': 'GET', 'PATH_INFO': '/foo/bar/'}).path) + self.assertEqual( + ['foo', 'bar'], + Request({'REQUEST_METHOD': 'GET', 'PATH_INFO': '///foo////bar////'}).path) + + def test_Request_FailOnRelativePaths(self): + self.assertRaises(RuntimeError, Request, {'REQUEST_METHOD': 'GET', 'PATH_INFO': '..'}) + self.assertRaises(RuntimeError, Request, {'REQUEST_METHOD': 'GET', 'PATH_INFO': '/..'}) + self.assertRaises(RuntimeError, Request, {'REQUEST_METHOD': 'GET', 'PATH_INFO': '/../'}) + self.assertRaises(RuntimeError, Request, {'REQUEST_METHOD': 'GET', 'PATH_INFO': '../bar'}) + self.assertRaises(RuntimeError, Request, {'REQUEST_METHOD': 'GET', 'PATH_INFO': '/foo/../bar'}) + self.assertRaises(RuntimeError, Request, {'REQUEST_METHOD': 'GET', 'PATH_INFO': '/foo/..'}) + + def test_Request_EmptyArguments(self): + request = Request({'QUERY_STRING': 'a&b&c', 'PATH_INFO': '/', 'REQUEST_METHOD': 'GET'}) + self.assertEqual('', request['a']) + self.assertEqual('', request['b']) + self.assertEqual('', request['c']) + + def test_Request_UpdateQueryOnSets(self): + request = Request({'QUERY_STRING': 'a&b=2&c', 'PATH_INFO': '/', 'REQUEST_METHOD': 'GET'}) + self.assertEqual('a&b=2&c', request.query) + + request['a'] = 'a' + self.assertEqual('a=a&c=&b=2', request.query) + + request['b'] = 'b' + self.assertEqual('a=a&c=&b=b', request.query) + + request['c'] = 'c' + self.assertEqual('a=a&c=c&b=b', request.query) + + def test_parse_accept_language(self): + self.assertEqual( + ['ru', 'en', 'es'], + _parse_accept_language(' ru , en , es')) + self.assertEqual( + ['ru', 'en', 'es'], + _parse_accept_language(' en;q=.4 , ru, es;q=0.1')) + self.assertEqual( + ['ru', 'en', 'es'], + _parse_accept_language('ru;q=1,en;q=1,es;q=0.5')) + self.assertEqual( + ['ru-ru', 'es-br'], + _parse_accept_language('ru-RU,es_BR')) + + def test_JsonpCallback(self): + + class CommandsProcessor(object): + + @route('GET') + def get(self, request): + return 'ok' + + router = Router(CommandsProcessor()) + + response = [] + reply = router({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'GET', + 'QUERY_STRING': 'callback=foo', + }, + lambda status, headers: response.extend([status, dict(headers)])) + result = 'foo("ok");' + self.assertEqual(result, ''.join([i for i in reply])) + self.assertEqual([ + '200 OK', + {'content-length': str(len(result))}, + ], + response) + + def test_filename(self): + self.assertEqual('Foo', _filename('foo', None)) + self.assertEqual('Foo-Bar', _filename(['foo', 'bar'], None)) + self.assertEqual('FOO-BaR', _filename([' f o o', ' ba r '], None)) + + self.assertEqual('Foo-3', _filename(['foo', 3], None)) + + self.assertEqual('12-3', _filename(['/1/2/', '/3/'], None)) + + self.assertEqual('Foo.png', _filename('foo', 'image/png')) + self.assertEqual('Foo-Bar.gif', _filename(['foo', 'bar'], 'image/gif')) + self.assertEqual('Fake', _filename('fake', 'foo/bar')) + + self.assertEqual('Eng', _filename({default_lang(): 'eng'}, None)) + self.assertEqual('Eng', _filename([{default_lang(): 'eng'}], None)) + self.assertEqual('Bar-1', _filename([{'lang': 'foo', default_lang(): 'bar'}, 1], None)) + + def test_BlobsDisposition(self): + self.touch(('blob.data', 'value')) + + class CommandsProcessor(object): + + @route('GET', [], '1') + def cmd1(self, request): + return Blob(name='foo', blob='blob.data') + + @route('GET', [], cmd='2') + def cmd2(self, request): + return Blob(filename='foo.bar', blob='blob.data') + + router = Router(CommandsProcessor()) + + response = [] + reply = router({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'GET', + 'QUERY_STRING': 'cmd=1', + }, + lambda status, headers: response.extend([status, dict(headers)])) + result = 'value' + self.assertEqual(result, ''.join([i for i in reply])) + self.assertEqual([ + '200 OK', + { + 'last-modified': formatdate(os.stat('blob.data').st_mtime, localtime=False, usegmt=True), + 'content-length': str(len(result)), + 'content-type': 'application/octet-stream', + 'content-disposition': 'attachment; filename="Foo.obj"', + } + ], + response) + + response = [] + reply = router({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'GET', + 'QUERY_STRING': 'cmd=2', + }, + lambda status, headers: response.extend([status, dict(headers)])) + result = 'value' + self.assertEqual(result, ''.join([i for i in reply])) + self.assertEqual([ + '200 OK', + { + 'last-modified': formatdate(os.stat('blob.data').st_mtime, localtime=False, usegmt=True), + 'content-length': str(len(result)), + 'content-type': 'application/octet-stream', + 'content-disposition': 'attachment; filename="foo.bar"', + } + ], + response) + + def test_DoNotOverrideContentLengthForHEAD(self): + + class CommandsProcessor(object): + + @route('HEAD', []) + def head(self, request, response): + response.content_length = 100 + + router = Router(CommandsProcessor()) + + response = [] + reply = router({ + 'PATH_INFO': '/', + 'REQUEST_METHOD': 'HEAD', + }, + lambda status, headers: response.extend([status, dict(headers)])) + self.assertEqual([], [i for i in reply]) + self.assertEqual([ + '200 OK', + {'content-length': '100'}, + ], + response) + + +if __name__ == '__main__': + tests.main() diff --git a/tests/units/toolkit/util.py b/tests/units/toolkit/toolkit.py index ada713d..b901583 100755 --- a/tests/units/toolkit/util.py +++ b/tests/units/toolkit/toolkit.py @@ -7,8 +7,8 @@ from cStringIO import StringIO from __init__ import tests -from sugar_network.toolkit import util -from sugar_network.toolkit.util import Seqno, Sequence +from sugar_network import toolkit +from sugar_network.toolkit import Seqno, Sequence class UtilTest(tests.Test): @@ -350,7 +350,7 @@ class UtilTest(tests.Test): result = [] stream = StringIO(string) while True: - line = util.readline(stream) + line = toolkit.readline(stream) if not line: break result.append(line) @@ -364,39 +364,39 @@ class UtilTest(tests.Test): self.assertEqual([' \n', ' b \n'], readlines(' \n b \n')) def test_Pool(self): - stack = util.Pool() + stack = toolkit.Pool() stack.add('a') stack.add('b') stack.add('c') - self.assertEqual(util.Pool.QUEUED, stack.get_state('a')) - self.assertEqual(util.Pool.QUEUED, stack.get_state('b')) - self.assertEqual(util.Pool.QUEUED, stack.get_state('c')) + self.assertEqual(toolkit.Pool.QUEUED, stack.get_state('a')) + self.assertEqual(toolkit.Pool.QUEUED, stack.get_state('b')) + self.assertEqual(toolkit.Pool.QUEUED, stack.get_state('c')) self.assertEqual( - [('c', util.Pool.ACTIVE), ('b', util.Pool.ACTIVE), ('a', util.Pool.ACTIVE)], + [('c', toolkit.Pool.ACTIVE), ('b', toolkit.Pool.ACTIVE), ('a', toolkit.Pool.ACTIVE)], [(i, stack.get_state(i)) for i in stack]) self.assertEqual( [], [i for i in stack]) - self.assertEqual(util.Pool.PASSED, stack.get_state('a')) - self.assertEqual(util.Pool.PASSED, stack.get_state('b')) - self.assertEqual(util.Pool.PASSED, stack.get_state('c')) + self.assertEqual(toolkit.Pool.PASSED, stack.get_state('a')) + self.assertEqual(toolkit.Pool.PASSED, stack.get_state('b')) + self.assertEqual(toolkit.Pool.PASSED, stack.get_state('c')) stack.rewind() - self.assertEqual(util.Pool.QUEUED, stack.get_state('a')) - self.assertEqual(util.Pool.QUEUED, stack.get_state('b')) - self.assertEqual(util.Pool.QUEUED, stack.get_state('c')) + self.assertEqual(toolkit.Pool.QUEUED, stack.get_state('a')) + self.assertEqual(toolkit.Pool.QUEUED, stack.get_state('b')) + self.assertEqual(toolkit.Pool.QUEUED, stack.get_state('c')) self.assertEqual( ['c', 'b', 'a'], [i for i in stack]) stack.add('c') - self.assertEqual(util.Pool.QUEUED, stack.get_state('c')) + self.assertEqual(toolkit.Pool.QUEUED, stack.get_state('c')) self.assertEqual( - [('c', util.Pool.ACTIVE)], + [('c', toolkit.Pool.ACTIVE)], [(i, stack.get_state(i)) for i in stack]) - self.assertEqual(util.Pool.PASSED, stack.get_state('c')) + self.assertEqual(toolkit.Pool.PASSED, stack.get_state('c')) stack.add('b') stack.add('a') @@ -410,17 +410,45 @@ class UtilTest(tests.Test): [i for i in stack]) stack.add('d') - self.assertEqual(util.Pool.QUEUED, stack.get_state('d')) + self.assertEqual(toolkit.Pool.QUEUED, stack.get_state('d')) self.assertEqual( - [('d', util.Pool.ACTIVE)], + [('d', toolkit.Pool.ACTIVE)], [(i, stack.get_state(i)) for i in stack]) - self.assertEqual(util.Pool.PASSED, stack.get_state('d')) + self.assertEqual(toolkit.Pool.PASSED, stack.get_state('d')) stack.rewind() self.assertEqual( ['d', 'a', 'b', 'c'], [i for i in stack]) + def test_gettext(self): + # Fallback to default lang + toolkit._default_lang = 'default' + self.assertEqual('foo', toolkit.gettext({'lang': 'foo', 'default': 'bar'}, 'lang')) + self.assertEqual('bar', toolkit.gettext({'lang': 'foo', 'default': 'bar'}, 'fake')) + + # Exact accept_language + self.assertEqual('', toolkit.gettext(None, 'lang')) + self.assertEqual('foo', toolkit.gettext('foo', 'lang')) + self.assertEqual('foo', toolkit.gettext({'lang': 'foo', 'fake': 'bar', 'default': 'default'}, 'lang')) + self.assertEqual('foo', toolkit.gettext({'lang': 'foo', 'fake': 'bar', 'default': 'default'}, ['lang', 'fake'])) + self.assertEqual('bar', toolkit.gettext({'lang': 'foo', 'fake': 'bar', 'default': 'default'}, ['fake', 'lang'])) + + # Last resort + self.assertEqual('foo', toolkit.gettext({'1': 'foo', '2': 'bar'}, 'fake')) + + # Primed accept_language + self.assertEqual('foo', toolkit.gettext({'1': 'foo', '2': 'bar', 'default': 'default'}, '1-a')) + + # Primed i18n value + self.assertEqual('bar', toolkit.gettext({'1-a': 'foo', '1': 'bar', 'default': 'default'}, '1-b')) + self.assertEqual('foo', toolkit.gettext({'1-a': 'foo', '2': 'bar', 'default': 'default'}, '1-b')) + + def test_gettext_EnAsTheLastResort(self): + toolkit._default_lang = 'en-us' + self.assertEqual('right', toolkit.gettext({'a': 'wrong', 'en': 'right'}, 'probe')) + self.assertEqual('exact', toolkit.gettext({'a': 'wrong', 'en': 'right', 'probe': 'exact'}, 'probe')) + if __name__ == '__main__': tests.main() |