diff options
author | Aleksey Lim <alsroot@sugarlabs.org> | 2013-06-09 06:10:45 (GMT) |
---|---|---|
committer | Aleksey Lim <alsroot@sugarlabs.org> | 2013-06-09 06:10:45 (GMT) |
commit | 71af69c6dc81aa480fc841f2f8d3ab0981085347 (patch) | |
tree | bc1a4cd901d3cad861d3bd96d0529e228b6d3722 | |
parent | 7aa45464388ee79355cb6e54730b3c8c6e8b571d (diff) |
Polish internal implementation
* Keep only low level storage logic in non-commands db code;
* Do not send props in events.
33 files changed, 853 insertions, 918 deletions
diff --git a/misc/aslo-sync b/misc/aslo-sync index 02da0ab..90364e0 100755 --- a/misc/aslo-sync +++ b/misc/aslo-sync @@ -161,24 +161,26 @@ class Application(application.Application): 'pull activities.sugarlabs.org content to local db') def pull(self): if not self.volume['context'].exists(SN_GUID): - self.volume['context'].create( - guid=SN_GUID, - implement=SN_GUID, - type='project', - title='Sugar Network', - summary='Sugar Network', - description='Sugar Network', - ctime=time.time(), mtime=time.time(), - author=self.authors(), - ) + self.volume['context'].create({ + 'guid': SN_GUID, + 'implement': SN_GUID, + 'type': 'project', + 'title': 'Sugar Network', + 'summary': 'Sugar Network', + 'description': 'Sugar Network', + 'ctime': time.time(), + 'mtime': time.time(), + 'author': self.authors(), + }) if not self.volume['context'].exists(SUGAR_GUID): - self.volume['context'].create( - guid=SUGAR_GUID, - implement=SUGAR_GUID, - type='package', title='sugar', - summary='Constructionist learning platform', - description= + self.volume['context'].create({ + 'guid': SUGAR_GUID, + 'implement': SUGAR_GUID, + 'type': 'package', + 'title': 'sugar', + 'summary': 'Constructionist learning platform', + 'description': 'Sugar provides simple yet powerful means of engaging ' 'young children in the world of learning that is ' 'opened up by computers and the Internet. With Sugar, ' @@ -188,21 +190,23 @@ class Application(application.Application): 'sharing, collaborative learning, and reflection, ' 'developing skills that help them in all aspects ' 'of life.', - ctime=time.time(), mtime=time.time(), - author=self.authors(), - ) + 'ctime': time.time(), + 'mtime': time.time(), + 'author': self.authors(), + }) if not self.volume['context'].exists(PACKAGES_GUID): - self.volume['context'].create( - guid=PACKAGES_GUID, - implement=PACKAGES_GUID, - type='project', - title='Packages', - summary='Collection of GNU/Linux packages metadata', - description='Collection of GNU/Linux packages metadata', - ctime=time.time(), mtime=time.time(), - author=self.authors(), - ) + self.volume['context'].create({ + 'guid': PACKAGES_GUID, + 'implement': PACKAGES_GUID, + 'type': 'project', + 'title': 'Packages', + 'summary': 'Collection of GNU/Linux packages metadata', + 'description': 'Collection of GNU/Linux packages metadata', + 'ctime': time.time(), + 'mtime': time.time(), + 'author': self.authors(), + }) self.volume['context'].set_blob(PACKAGES_GUID, 'icon', url='/static/images/package.png') @@ -305,16 +309,16 @@ class Application(application.Application): self.sqlexec(sql): if directory.exists(str(guid)): continue - directory.create( - guid=str(guid), - ctime=int(time.mktime(created.timetuple())), - mtime=int(time.mktime(created.timetuple())), - context=bundle_id, - title=self.get_i18n_field(title), - content=self.get_i18n_field(content), - rating=rating, - author=self.authors(nickname, author), - ) + directory.create({ + 'guid': str(guid), + 'ctime': int(time.mktime(created.timetuple())), + 'mtime': int(time.mktime(created.timetuple())), + 'context': bundle_id, + 'title': self.get_i18n_field(title), + 'content': self.get_i18n_field(content), + 'rating': rating, + 'author': self.authors(nickname, author), + }) def sync_versions(self, addon_id, bundle_id): sql = """ @@ -484,13 +488,22 @@ class Application(application.Application): shutil.rmtree(tmpdir) print '-- Update %r metadata from %r' % (bundle_id, filename) - self.volume['context'].update(bundle_id, **props) + self.volume['context'].update(bundle_id, props) def sync_context(self, addon_id, bundle_id): if not self.volume['context'].exists(bundle_id): - self.volume['context'].create(guid=bundle_id, type='activity', - implement=bundle_id, title={}, summary={}, description={}, - author=self.authors(), layer=['public'], ctime=0, mtime=0) + self.volume['context'].create({ + 'guid': bundle_id, + 'type': 'activity', + 'implement': bundle_id, + 'title': {}, + 'summary': {}, + 'description': {}, + 'author': self.authors(), + 'layer': ['public'], + 'ctime': 0, + 'mtime': 0, + }) created, modified, title, summary, description, homepage, nickname, \ author = self.sqlexec(""" @@ -543,15 +556,16 @@ class Application(application.Application): """ % addon_id): tags.add(row[0]) - self.volume['context'].update(bundle_id, - title=self.get_i18n_field(title), - summary=self.get_i18n_field(summary), - description=self.get_i18n_field(description), - homepage=homepage or '', - tags=list(tags), - author=self.authors(nickname, author), - ctime=created, - mtime=modified) + self.volume['context'].update(bundle_id, { + 'title': self.get_i18n_field(title), + 'summary': self.get_i18n_field(summary), + 'description': self.get_i18n_field(description), + 'homepage': homepage or '', + 'tags': list(tags), + 'author': self.authors(nickname, author), + 'ctime': created, + 'mtime': modified, + }) def sync_implementaiton(self, context, addon_id, filename, sugar_min, sugar_max, **impl_props): @@ -584,18 +598,20 @@ class Application(application.Application): if release >= sugar_min and release <= sugar_max: requires.append(name) - impl = self.volume['implementation'].create( - context=context, - version=spec['version'], - requires=requires, - spec={'*-*': { + impl = {'context': context, + 'version': spec['version'], + 'requires': requires, + 'spec': {'*-*': { 'commands': spec.commands, 'requires': spec.requires, 'extract': bundle.rootdir, }}, - ctime=time.time(), mtime=time.time(), - author=self.authors(), - **impl_props) + 'ctime': time.time(), + 'mtime': time.time(), + 'author': self.authors(), + } + impl.update(impl_props) + impl = self.volume['implementation'].create(impl) self.volume['implementation'].set_blob(impl, 'data', url='/'.join([DOWNLOAD_URL, str(addon_id), filename])) diff --git a/sugar_network/client/clones.py b/sugar_network/client/clones.py index d21ffb0..ac44fe6 100644 --- a/sugar_network/client/clones.py +++ b/sugar_network/client/clones.py @@ -111,9 +111,9 @@ class _Inotify(Inotify): break if found: if context['clone'] != 2: - self._contexts.update(context.guid, clone=2) + self._contexts.update(context.guid, {'clone': 2}) else: - self._contexts.update(context.guid, clone=0) + self._contexts.update(context.guid, {'clone': 0}) def serve_forever(self): while True: @@ -156,17 +156,25 @@ class _Inotify(Inotify): _logger.debug('Register unknown local activity, %r', context) mtime = os.stat(spec.root).st_mtime - self._contexts.create(guid=context, type='activity', - title=spec['name'], summary=spec['summary'], - description=spec['description'], clone=2, - ctime=mtime, mtime=mtime) + self._contexts.create({ + 'guid': context, + 'type': 'activity', + 'title': spec['name'], + 'summary': spec['summary'], + 'description': spec['description'], + 'clone': 2, + 'ctime': mtime, + 'mtime': mtime, + }) icon_path = join(spec.root, spec['icon']) if exists(icon_path): - self._contexts.set_blob(context, 'artifact_icon', icon_path) + with file(icon_path, 'b') as f: + self._contexts.update(context, + {'artifact_icon': {'blob': f}}) with util.NamedTemporaryFile() as f: util.svg_to_png(icon_path, f.name, 32, 32) - self._contexts.set_blob(context, 'icon', f.name) + self._contexts.update(context, {'icon': {'blob': f.name}}) self._checkin_activity(spec) diff --git a/sugar_network/client/commands.py b/sugar_network/client/commands.py index 92cf488..f69d8cb 100644 --- a/sugar_network/client/commands.py +++ b/sugar_network/client/commands.py @@ -67,7 +67,7 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): static_prefix = 'http://127.0.0.1:%s' % client.ipc_port.value self._static_prefix = static_prefix - home_volume.connect(self._home_event_cb) + home_volume.connect(self.broadcast) if self._server_mode: mountpoints.connect(_SN_DIRNAME, @@ -369,7 +369,7 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): def pull_events(): for event in self._node.subscribe(): if event.get('document') == 'implementation': - mtime = event.get('props', {}).get('mtime') + mtime = event.get('mtime') if mtime: injector.invalidate_solutions(mtime) self.broadcast(event) @@ -447,19 +447,6 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): self._inline_job.kill() self._got_offline() - def _home_event_cb(self, event): - if not self._inline.is_set(): - self.broadcast(event) - elif event.get('document') == 'context' and 'props' in event: - # Broadcast events related to proxy properties - event_props = event['props'] - broadcast_props = event['props'] = {} - for name in _LOCAL_PROPS: - if name in event_props: - broadcast_props[name] = event_props[name] - if broadcast_props: - self.broadcast(event) - def _clone_jobject(self, uid, value, get_props, force): if value: if force or not journal.exists(uid): @@ -487,7 +474,7 @@ class ClientCommands(db.CommandsProcessor, Commands, journal.Commands): blob = self._node_call(method='GET', document='context', guid=guid, prop=prop) if blob is not None: - contexts.set_blob(guid, prop, blob) + contexts.update(guid, {prop: {'blob': blob}}) def _clone_activity(self, guid, request): if not request.content: @@ -676,9 +663,9 @@ class _VolumeCommands(db.VolumeCommands): def __init__(self, volume): db.VolumeCommands.__init__(self, volume) - def before_create(self, request, props): + def on_create(self, request, props, event): props['layer'] = tuple(props['layer']) + ('local',) - db.VolumeCommands.before_create(self, request, props) + db.VolumeCommands.on_create(self, request, props, event) class _PersonalCommands(SlaveCommands): @@ -693,7 +680,9 @@ class _PersonalCommands(SlaveCommands): users = volume['user'] if not users.exists(client.sugar_uid()): - users.create(guid=client.sugar_uid(), **client.sugar_profile()) + profile = client.sugar_profile() + profile['guid'] = client.sugar_uid() + users.create(profile) mountpoints.connect(_SYNC_DIRNAME, self.__found_mountcb, self.__lost_mount_cb) diff --git a/sugar_network/db/directory.py b/sugar_network/db/directory.py index 8eb36d4..a4f446e 100644 --- a/sugar_network/db/directory.py +++ b/sugar_network/db/directory.py @@ -71,7 +71,7 @@ class Directory(object): @mtime.setter def mtime(self, value): self._index.mtime = value - self._notify({'event': 'populate', 'props': {'mtime': value}}) + self._notify({'event': 'populate', 'mtime': value}) def wipe(self): self.close() @@ -91,39 +91,29 @@ class Directory(object): """Flush pending chnages to disk.""" self._index.commit() - def create(self, props=None, **kwargs): + def create(self, props, event=None): """Create new document. If `guid` property is not specified, it will be auto set. - :param kwargs: + :param props: new document properties :returns: GUID of newly created document """ - if props is None: - props = kwargs - guid = props.get('guid') if not guid: guid = props['guid'] = toolkit.uuid() - - for prop_name, prop in self.metadata.items(): - if isinstance(prop, StoredProperty): - if prop_name in props: - continue - enforce(prop.default is not None, - 'Property %r should be passed for new %r document', - prop_name, self.metadata.name) - if prop.default is not None: - props[prop_name] = prop.default - _logger.debug('Create %s[%s]: %r', self.metadata.name, guid, props) - self._post(guid, props, True) + post_event = {'event': 'create', 'guid': guid} + if event: + post_event.update(event) + self._index.store(guid, props, self._pre_store, self._post_store, + post_event) return guid - def update(self, guid, props=None, **kwargs): + def update(self, guid, props, event=None): """Update properties for an existing document. :param guid: @@ -132,12 +122,12 @@ class Directory(object): properties to store, not necessary all document's properties """ - if props is None: - props = kwargs - if not props: - return _logger.debug('Update %s[%s]: %r', self.metadata.name, guid, props) - self._post(guid, props, False) + post_event = {'event': 'update', 'guid': guid} + if event: + post_event.update(event) + self._index.store(guid, props, self._pre_store, self._post_store, + post_event) def delete(self, guid): """Delete document. @@ -206,36 +196,6 @@ class Directory(object): return iterate(), mset.get_matches_estimated() - def set_blob(self, guid, prop, data=None, size=None, mime_type=None, - **kwargs): - """Receive BLOB property. - - This function works in parallel to setting non-BLOB properties values - and `post()` function. - - :param prop: - BLOB property name - :param data: - stream to read BLOB content, path to file to copy, or, web url - :param size: - read only specified number of bytes; otherwise, read until the EOF - - """ - prop = self.metadata[prop] - record = self._storage.get(guid) - seqno = self._seqno.next() - - _logger.debug('Received %r BLOB property from %s[%s]', - prop.name, self.metadata.name, guid) - - if not mime_type: - mime_type = prop.mime_type - record.set_blob(prop.name, data, size, seqno=seqno, - mime_type=mime_type, **kwargs) - - if record.consistent: - self._post(guid, {'seqno': seqno}, False) - def populate(self): """Populate the index. @@ -249,9 +209,9 @@ class Directory(object): """ found = False - migrate = (self._index.mtime == 0) + migrate = (self.mtime == 0) - for guid in self._storage.walk(self._index.mtime): + for guid in self._storage.walk(self.mtime): if not found: _logger.info('Start populating %r index', self.metadata.name) found = True @@ -268,7 +228,7 @@ class Directory(object): meta = record.get(name) if meta is not None: props[name] = meta['value'] - self._index.store(guid, props, None, None, None) + self._index.store(guid, props) yield except Exception: exception('Cannot populate %r in %r, invalidate it', @@ -279,7 +239,7 @@ class Directory(object): self._index.checkpoint() self._save_layout() self.commit() - self._notify({'event': 'populate'}) + self._notify({'event': 'populate', 'mtime': self.mtime}) def diff(self, seq, exclude_seq=None, **params): if exclude_seq is None: @@ -347,11 +307,9 @@ class Directory(object): props = {} if seqno: props['seqno'] = seqno - self._index.store(guid, props, False, - self._pre_store, self._post_store, - # No need in after-merge event, further commit event - # is enough to avoid events flow on nodes synchronization - None, False) + # No need in after-merge event, further commit event + # is enough to avoid events flow on nodes synchronization + self._index.store(guid, props, self._pre_store, self._post_store) return seqno, merged @@ -370,39 +328,41 @@ class Directory(object): self._post_commit) _logger.debug('Initiated %r document', self.document_class) - def _pre_store(self, guid, changes, event, shift_seqno): + def _pre_store(self, guid, changes, event=None): seqno = changes.get('seqno') - if shift_seqno and not seqno: + if event is not None and not seqno: seqno = changes['seqno'] = self._seqno.next() record = self._storage.get(guid) existed = record.exists for name, prop in self.metadata.items(): - if not isinstance(prop, StoredProperty): - continue value = changes.get(name) - if value is None: - if existed: - meta = record.get(name) - if meta is not None: - value = meta['value'] - changes[name] = prop.default if value is None else value - else: - if prop.localized: - if not isinstance(value, dict): - value = {toolkit.default_lang(): value} - if existed and \ - type(value) is dict: # TODO To reset `value` + if isinstance(prop, BlobProperty): + if value is not None: + record.set(name, seqno=seqno, **value) + elif isinstance(prop, StoredProperty): + if value is None: + if existed: meta = record.get(name) if meta is not None: - meta['value'].update(value) value = meta['value'] - changes[name] = value - record.set(name, value=value, seqno=seqno) - - def _post_store(self, guid, changes, event, shift_seqno): - if event: + changes[name] = prop.default if value is None else value + else: + if prop.localized: + if not isinstance(value, dict): + value = {toolkit.default_lang(): value} + if existed and \ + type(value) is dict: # TODO To reset `value` + meta = record.get(name) + if meta is not None: + meta['value'].update(value) + value = meta['value'] + changes[name] = value + record.set(name, value=value, seqno=seqno) + + def _post_store(self, guid, changes, event=None): + if event is not None: self._notify(event) def _post_delete(self, guid, event): @@ -411,15 +371,7 @@ class Directory(object): def _post_commit(self): self._seqno.commit() - self._notify({'event': 'commit'}) - - def _post(self, guid, props, new): - event = {'event': 'create' if new else 'update', - 'props': props.copy(), - 'guid': guid, - } - self._index.store(guid, props, new, - self._pre_store, self._post_store, event, True) + self._notify({'event': 'commit', 'mtime': self.mtime}) def _notify(self, event): if self._notification_cb is not None: diff --git a/sugar_network/db/document.py b/sugar_network/db/document.py index f8a458f..bdc9660 100644 --- a/sugar_network/db/document.py +++ b/sugar_network/db/document.py @@ -35,6 +35,7 @@ class Document(object): self.is_new = not bool(guid) self._record = record self.request = request + self._modifies = set() @property def volume(self): @@ -76,6 +77,8 @@ class Document(object): if isinstance(prop, StoredProperty): if meta is not None: value = meta.get('value') + else: + value = prop.default else: value = meta or PropertyMetadata() self.props[prop.name] = value @@ -95,8 +98,12 @@ class Document(object): def meta(self, prop): return self._record.get(prop) + def modified(self, prop): + return prop in self._modifies + def __getitem__(self, prop): return self.get(prop) def __setitem__(self, prop, value): self.props[prop] = value + self._modifies.add(prop) diff --git a/sugar_network/db/index.py b/sugar_network/db/index.py index 2fc7ef7..a87ffdc 100644 --- a/sugar_network/db/index.py +++ b/sugar_network/db/index.py @@ -80,7 +80,7 @@ class IndexReader(object): """ pass - def store(self, guid, properties, new, pre_cb=None, post_cb=None, *args): + def store(self, guid, properties, pre_cb=None, post_cb=None, *args): """Store new document in the index. :param guid: @@ -88,8 +88,6 @@ class IndexReader(object): :param properties: document's properties to store; for non new entities, not necessary all document's properties - :param new: - initial store for the document; `None` for merging from other nodes :param pre_cb: callback to execute before storing; will be called with passing `guid` and `properties` @@ -315,7 +313,7 @@ class IndexWriter(IndexReader): self._do_open() return IndexReader.find(self, query) - def store(self, guid, properties, new, pre_cb=None, post_cb=None, *args): + def store(self, guid, properties, pre_cb=None, post_cb=None, *args): if self._db is None: self._do_open() diff --git a/sugar_network/db/metadata.py b/sugar_network/db/metadata.py index b32cc54..a5ba8c3 100644 --- a/sugar_network/db/metadata.py +++ b/sugar_network/db/metadata.py @@ -128,11 +128,6 @@ class PropertyMetadata(dict): meta['mtime'] = int(os.stat(path_).st_mtime) dict.__init__(self, meta) - @classmethod - def is_blob(cls, blob): - return isinstance(blob, (type(None), basestring, cls)) or \ - hasattr(blob, 'read') - class Property(object): """Basic class to collect information about document property.""" diff --git a/sugar_network/db/router.py b/sugar_network/db/router.py index bb8f701..e3e56eb 100644 --- a/sugar_network/db/router.py +++ b/sugar_network/db/router.py @@ -72,7 +72,7 @@ class Router(object): request = Request(method='GET', cmd='exists', document='user', guid=user) enforce(self.commands.call(request), http.Unauthorized, - 'Principal user does not exist') + 'Principal does not exist') self._authenticated.add(user) return user diff --git a/sugar_network/db/storage.py b/sugar_network/db/storage.py index 70e97b0..8416dee 100644 --- a/sugar_network/db/storage.py +++ b/sugar_network/db/storage.py @@ -14,16 +14,14 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import os -import sys import time import json import shutil -import hashlib import cPickle as pickle -from os.path import exists, join, isdir, basename, relpath, isabs +from os.path import exists, join, isdir, basename from sugar_network.db.metadata import PropertyMetadata, BlobProperty -from sugar_network.toolkit import BUFFER_SIZE, util, exception +from sugar_network.toolkit import util, exception class Storage(object): @@ -131,20 +129,21 @@ class Record(object): if exists(path): return PropertyMetadata(path) - def set(self, prop, mtime=None, path=None, blob=None, **meta): + def set(self, prop, mtime=None, **meta): if not exists(self._root): os.makedirs(self._root) meta_path = join(self._root, prop) - if isinstance(blob, basestring): - path = blob - blob = None - blob_path = join(self._root, prop + PropertyMetadata.BLOB_SUFFIX) - if blob is not None: - with util.new_file(blob_path) as f: - shutil.copyfileobj(blob, f) - elif path and exists(path): - util.cptree(path, blob_path) + if 'blob' in meta: + dst_blob_path = meta_path + PropertyMetadata.BLOB_SUFFIX + blob = meta.pop('blob') + if hasattr(blob, 'read'): + with util.new_file(dst_blob_path) as f: + shutil.copyfileobj(blob, f) + elif blob is not None: + os.rename(blob, dst_blob_path) + elif exists(dst_blob_path): + os.unlink(dst_blob_path) with util.new_file(meta_path) as f: json.dump(meta, f) @@ -157,64 +156,3 @@ class Record(object): # Touch directory to let it possible to crawl it on startup # when index was not previously closed properly os.utime(join(self._root, '..'), (mtime, mtime)) - - def set_blob(self, prop, data=None, size=None, **kwargs): - if not exists(self._root): - os.makedirs(self._root) - path = join(self._root, prop + PropertyMetadata.BLOB_SUFFIX) - meta = PropertyMetadata(**kwargs) - - if data is None: - if exists(path): - os.unlink(path) - elif isinstance(data, PropertyMetadata): - data.update(meta) - meta = data - else: - digest = hashlib.sha1() - if hasattr(data, 'read'): - if size is None: - size = sys.maxint - self._set_blob_by_stream(digest, data, size, path) - elif isabs(data) and exists(data): - self._set_blob_by_path(digest, data, path) - else: - with util.new_file(path) as f: - f.write(data) - digest.update(data) - meta['digest'] = digest.hexdigest() - - self.set(prop, **meta) - - def _set_blob_by_stream(self, digest, stream, size, path): - with util.new_file(path) as f: - while size > 0: - chunk = stream.read(min(size, BUFFER_SIZE)) - if not chunk: - break - f.write(chunk) - size -= len(chunk) - if digest is not None: - digest.update(chunk) - - def _set_blob_by_path(self, digest, src_path, dst_path): - util.cptree(src_path, dst_path) - - def hash_file(path): - with file(path) as f: - while True: - chunk = f.read(BUFFER_SIZE) - if not chunk: - break - if digest is not None: - digest.update(chunk) - - if isdir(dst_path): - for root, __, files in os.walk(dst_path): - for filename in files: - path = join(root, filename) - if digest is not None: - digest.update(relpath(path, dst_path)) - hash_file(path) - else: - hash_file(dst_path) diff --git a/sugar_network/db/volume.py b/sugar_network/db/volume.py index 8c248b0..1408a83 100644 --- a/sugar_network/db/volume.py +++ b/sugar_network/db/volume.py @@ -15,7 +15,9 @@ import os import re +import sys import time +import hashlib import logging from contextlib import contextmanager from os.path import exists, join, abspath @@ -30,7 +32,7 @@ from sugar_network.db.commands import to_int, to_list from sugar_network.db.metadata import BlobProperty, StoredProperty from sugar_network.db.metadata import PropertyMetadata from sugar_network.toolkit import http, coroutine, util -from sugar_network.toolkit import exception, enforce +from sugar_network.toolkit import BUFFER_SIZE, exception, enforce _GUID_RE = re.compile('[a-zA-Z0-9_+-.]+$') @@ -132,11 +134,12 @@ class VolumeCommands(CommandsProcessor): permissions=env.ACCESS_AUTH, mime_type='application/json') def create(self, request): with self._post(request, env.ACCESS_CREATE) as (directory, doc): - self.before_create(request, doc.props) + event = {} + self.on_create(request, doc.props, event) if 'guid' not in doc.props: doc.props['guid'] = toolkit.uuid() doc.guid = doc.props['guid'] - directory.create(doc.props) + directory.create(doc.props, event) return doc.guid @directory_command(method='GET', @@ -160,10 +163,11 @@ class VolumeCommands(CommandsProcessor): permissions=env.ACCESS_AUTH | env.ACCESS_AUTHOR) def update(self, request): with self._post(request, env.ACCESS_WRITE) as (directory, doc): - modified = bool(doc.props) - self.before_update(request, doc.props) - if modified: - directory.update(doc.guid, doc.props) + if not doc.props: + return + event = {} + self.on_update(request, doc.props, event) + directory.update(doc.guid, doc.props, event) @property_command(method='PUT', permissions=env.ACCESS_AUTH | env.ACCESS_AUTHOR) @@ -179,7 +183,7 @@ class VolumeCommands(CommandsProcessor): @document_command(method='DELETE', permissions=env.ACCESS_AUTH | env.ACCESS_AUTHOR) - def delete(self, document, guid): + def delete(self, request, document, guid): directory = self.volume[document] directory.delete(guid) @@ -217,7 +221,7 @@ class VolumeCommands(CommandsProcessor): http.NotFound, 'BLOB does not exist') return meta - def before_create(self, request, props): + def on_create(self, request, props, event): if 'guid' in props: # TODO Temporal security hole, see TODO guid = props['guid'] @@ -229,7 +233,7 @@ class VolumeCommands(CommandsProcessor): props['ctime'] = ts props['mtime'] = ts - def before_update(self, request, props): + def on_update(self, request, props, event): props['mtime'] = int(time.time()) def after_post(self, doc): @@ -249,49 +253,50 @@ class VolumeCommands(CommandsProcessor): for name, value in request.content.items(): prop = directory.metadata[name] - if isinstance(prop, BlobProperty) and access == env.ACCESS_WRITE: - if doc.meta(name) is None: - prop.assert_access(env.ACCESS_CREATE) + if isinstance(prop, BlobProperty): + prop.assert_access(env.ACCESS_CREATE if + access == env.ACCESS_WRITE and doc.meta(name) is None + else access) + if value is None: + value = {'blob': None} + elif isinstance(value, dict): + enforce('url' in value, + 'Key %r is not specified in %r blob property', + 'url', name) + value = {'url': value['url']} else: - prop.assert_access(env.ACCESS_WRITE) + value = _read_blob(request, prop, value) + blobs.append(value['blob']) else: prop.assert_access(access) - if isinstance(prop, BlobProperty): - enforce(PropertyMetadata.is_blob(value), 'Invalid BLOB value') - blobs.append((name, value)) - else: if prop.localized and isinstance(value, basestring): value = {request.accept_language[0]: value} try: - doc.props[name] = prop.decode(value) + value = prop.decode(value) except Exception, error: error = 'Value %r for %r property is invalid: %s' % \ (value, prop.name, error) exception(error) raise RuntimeError(error) + doc[name] = value if access == env.ACCESS_CREATE: for name, prop in directory.metadata.items(): if not isinstance(prop, BlobProperty) and \ name not in request.content and \ (prop.default is not None or prop.on_set is not None): - doc.props[name] = prop.default - - for name, value in doc.props.items(): - prop = directory.metadata[name] - if not isinstance(prop, BlobProperty) and prop.on_set is not None: - doc.props[name] = prop.on_set(doc, value) - - changed_props = doc.props.copy() - yield directory, doc - doc.props = changed_props - - for name, value in blobs: - prop = directory.metadata[name] - if prop.on_set is not None: - value = prop.on_set(doc, value) - directory.set_blob(doc.guid, name, value, - mime_type=request.content_type) + doc[name] = prop.default + + try: + for name, value in doc.props.items(): + prop = directory.metadata[name] + if prop.on_set is not None: + doc.props[name] = prop.on_set(doc, value) + yield directory, doc + finally: + for path in blobs: + if exists(path): + os.unlink(path) self.after_post(doc) @@ -321,3 +326,32 @@ class VolumeCommands(CommandsProcessor): value = request.static_prefix + value result[name] = value return result + + +def _read_blob(request, prop, value): + digest = hashlib.sha1() + dst = util.NamedTemporaryFile(delete=False) + + try: + if isinstance(value, basestring): + digest.update(value) + dst.write(value) + else: + size = request.content_length or sys.maxint + while size > 0: + chunk = value.read(min(size, BUFFER_SIZE)) + if not chunk: + break + dst.write(chunk) + size -= len(chunk) + digest.update(chunk) + except Exception: + os.unlink(dst.name) + raise + finally: + dst.close() + + return {'blob': dst.name, + 'digest': digest.hexdigest(), + 'mime_type': request.content_type or prop.mime_type, + } diff --git a/sugar_network/node/commands.py b/sugar_network/node/commands.py index 5cc39f4..0f443cb 100644 --- a/sugar_network/node/commands.py +++ b/sugar_network/node/commands.py @@ -148,11 +148,12 @@ class NodeCommands(db.VolumeCommands, Commands): @db.document_command(method='DELETE', permissions=db.ACCESS_AUTH | db.ACCESS_AUTHOR) - def delete(self, document, guid): + def delete(self, request, document, guid): # Servers data should not be deleted immediately # to let master-slave synchronization possible - directory = self.volume[document] - directory.update(guid, {'layer': ['deleted']}) + request['method'] = 'PUT' + request.content = {'layer': ['deleted']} + self.update(request) @db.document_command(method='PUT', cmd='attach', permissions=db.ACCESS_AUTH) @@ -307,10 +308,16 @@ class NodeCommands(db.VolumeCommands, Commands): return cmd - def before_create(self, request, props): + def on_create(self, request, props, event): if request['document'] == 'user': props['guid'], props['pubkey'] = _load_pubkey(props['pubkey']) - db.VolumeCommands.before_create(self, request, props) + db.VolumeCommands.on_create(self, request, props, event) + + def on_update(self, request, props, event): + db.VolumeCommands.on_update(self, request, props, event) + if 'deleted' in props.get('layer', []): + event['event'] = 'delete' + print '1>>>', request, props, event @db.directory_command_pre(method='GET') def _NodeCommands_find_pre(self, request): diff --git a/sugar_network/node/master.py b/sugar_network/node/master.py index 30f6ebd..3619520 100644 --- a/sugar_network/node/master.py +++ b/sugar_network/node/master.py @@ -142,8 +142,8 @@ class MasterCommands(NodeCommands): def after_post(self, doc): if doc.metadata.name == 'context': - shift_implementations = ('dependencies' in doc.props) - if 'aliases' in doc.props: + shift_implementations = doc.modified('dependencies') + if doc.modified('aliases'): # TODO Already launched job should be killed coroutine.spawn(self._resolve_aliases, doc) shift_implementations = True diff --git a/sugar_network/resources/implementation.py b/sugar_network/resources/implementation.py index 5cc41ec..c29a41a 100644 --- a/sugar_network/resources/implementation.py +++ b/sugar_network/resources/implementation.py @@ -95,5 +95,5 @@ class Implementation(Resource): def data(self, value): context = self.volume['context'].get(self['context']) if 'activity' in context['type']: - self.request.content_type = 'application/vnd.olpc-sugar' + value['mime_type'] = 'application/vnd.olpc-sugar' return value diff --git a/sugar_network/resources/volume.py b/sugar_network/resources/volume.py index f71b2dc..feb23ff 100644 --- a/sugar_network/resources/volume.py +++ b/sugar_network/resources/volume.py @@ -74,7 +74,7 @@ class Resource(db.Document): permissions=db.ACCESS_AUTH | db.ACCESS_AUTHOR) def useradd(self, user, role): enforce(user, "Argument 'user' is not specified") - self.directory.update(self.guid, author=self._useradd(user, role)) + self.directory.update(self.guid, {'author': self._useradd(user, role)}) @db.document_command(method='PUT', cmd='userdel', permissions=db.ACCESS_AUTH | db.ACCESS_AUTHOR) @@ -84,7 +84,7 @@ class Resource(db.Document): author = self['author'] enforce(user in author, 'No such user') del author[user] - self.directory.update(self.guid, author=author) + self.directory.update(self.guid, {'author': author}) @db.indexed_property(prefix='RL', typecast=[], default=['public']) def layer(self, value): @@ -146,14 +146,6 @@ class Volume(db.Volume): self._populators.kill() db.Volume.close(self) - def notify(self, event): - if event['event'] == 'update' and 'props' in event and \ - 'deleted' in event['props'].get('layer', []): - event['event'] = 'delete' - del event['props'] - - db.Volume.notify(self, event) - def _open(self, name, document): directory = db.Volume._open(self, name, document) self._populators.spawn(self._populate, directory) diff --git a/sugar_network/toolkit/util.py b/sugar_network/toolkit/util.py index 470fb4f..9620efa 100644 --- a/sugar_network/toolkit/util.py +++ b/sugar_network/toolkit/util.py @@ -387,6 +387,14 @@ def unique_filename(root, filename): return path +def TemporaryFile(*args, **kwargs): + if cachedir.value: + if not exists(cachedir.value): + os.makedirs(cachedir.value) + kwargs['dir'] = cachedir.value + return tempfile.TemporaryFile(*args, **kwargs) + + def NamedTemporaryFile(*args, **kwargs): if cachedir.value: if not exists(cachedir.value): diff --git a/tests/units/client/clones.py b/tests/units/client/clones.py index 626715c..b9178a1 100755 --- a/tests/units/client/clones.py +++ b/tests/units/client/clones.py @@ -162,10 +162,14 @@ class CloneTest(tests.Test): self.volume['context'], ['Activities']) coroutine.sleep() - self.volume['context'].create( - guid='org.sugarlabs.HelloWorld', type='activity', - title={'en': 'title'}, summary={'en': 'summary'}, - description={'en': 'description'}, user=[tests.UID]) + self.volume['context'].create({ + 'guid': 'org.sugarlabs.HelloWorld', + 'type': 'activity', + 'title': {'en': 'title'}, + 'summary': {'en': 'summary'}, + 'description': {'en': 'description'}, + 'user': [tests.UID], + }) os.makedirs('Activities/activity/activity') coroutine.sleep(1) @@ -216,10 +220,14 @@ class CloneTest(tests.Test): self.volume['context'], ['Activities']) coroutine.sleep() - self.volume['context'].create( - guid='org.sugarlabs.HelloWorld', type='activity', - title={'en': 'title'}, summary={'en': 'summary'}, - description={'en': 'description'}, user=[tests.UID]) + self.volume['context'].create({ + 'guid': 'org.sugarlabs.HelloWorld', + 'type': 'activity', + 'title': {'en': 'title'}, + 'summary': {'en': 'summary'}, + 'description': {'en': 'description'}, + 'user': [tests.UID], + }) self.touch(('activity/activity/activity.info', [ '[Activity]', @@ -247,10 +255,14 @@ class CloneTest(tests.Test): self.volume['context'], ['Activities']) coroutine.sleep() - self.volume['context'].create( - guid='org.sugarlabs.HelloWorld', type='activity', - title={'en': 'title'}, summary={'en': 'summary'}, - description={'en': 'description'}, user=[tests.UID]) + self.volume['context'].create({ + 'guid': 'org.sugarlabs.HelloWorld', + 'type': 'activity', + 'title': {'en': 'title'}, + 'summary': {'en': 'summary'}, + 'description': {'en': 'description'}, + 'user': [tests.UID], + }) self.touch(('activity/activity/activity.info', [ '[Activity]', @@ -305,10 +317,14 @@ class CloneTest(tests.Test): self.job = coroutine.spawn(clones.monitor, self.volume['context'], ['Activities']) - self.volume['context'].create( - guid='org.sugarlabs.HelloWorld', type='activity', - title={'en': 'title'}, summary={'en': 'summary'}, - description={'en': 'description'}, user=[tests.UID]) + self.volume['context'].create({ + 'guid': 'org.sugarlabs.HelloWorld', + 'type': 'activity', + 'title': {'en': 'title'}, + 'summary': {'en': 'summary'}, + 'description': {'en': 'description'}, + 'user': [tests.UID], + }) self.touch('Activities/activity/activity/icon.svg') self.touch(('Activities/activity/activity/mimetypes.xml', [ diff --git a/tests/units/client/offline_commands.py b/tests/units/client/offline_commands.py index 8879c81..69d5f2a 100755 --- a/tests/units/client/offline_commands.py +++ b/tests/units/client/offline_commands.py @@ -107,8 +107,6 @@ class OfflineCommandsTest(tests.Test): def read_events(): for event in ipc.subscribe(event='!commit'): - if 'props' in event: - event.pop('props') events.append(event) job = coroutine.spawn(read_events) coroutine.dispatch() diff --git a/tests/units/client/online_commands.py b/tests/units/client/online_commands.py index e7baff4..12c5570 100755 --- a/tests/units/client/online_commands.py +++ b/tests/units/client/online_commands.py @@ -429,8 +429,6 @@ class OnlineCommandsTest(tests.Test): def read_events(): for event in ipc.subscribe(event='!commit'): - if 'props' in event: - event.pop('props') events.append(event) job = coroutine.spawn(read_events) coroutine.dispatch(.1) @@ -1012,7 +1010,7 @@ class OnlineCommandsTest(tests.Test): bundle.close() ipc.request('PUT', ['implementation', impl, 'data'], file('bundle', 'rb').read()) - trigger = self.wait_for_events(ipc, event='update', document='context', guid=context1, props={'clone': 2}) + trigger = self.wait_for_events(ipc, event='update', document='context', guid=context1) ipc.put(['context', context1], 2, cmd='clone') trigger.wait() self.assertEqual( @@ -1025,7 +1023,7 @@ class OnlineCommandsTest(tests.Test): 'summary': 'summary', 'description': 'description', }) - trigger = self.wait_for_events(ipc, event='create', document='context', guid=context2, props={'clone': 0, 'favorite': True}) + trigger = self.wait_for_events(ipc, event='create', document='context', guid=context2) ipc.put(['context', context2], True, cmd='favorite') trigger.wait() self.assertEqual( @@ -1080,7 +1078,7 @@ class OnlineCommandsTest(tests.Test): ipc.get(cmd='inline') self.wait_for_events(ipc, event='inline', state='online').wait() guid = ipc.post(['document'], {}) - home_volume['document'].create(guid=guid) + home_volume['document'].create({'guid': guid}) ts = time.time() self.assertEqual('remote', ipc.get(['document', guid], cmd='sleep')) diff --git a/tests/units/client/server_commands.py b/tests/units/client/server_commands.py index 57becf2..771d008 100755 --- a/tests/units/client/server_commands.py +++ b/tests/units/client/server_commands.py @@ -78,8 +78,6 @@ class ServerCommandsTest(tests.Test): def read_events(): for event in ipc.subscribe(event='!commit'): - if 'props' in event: - event.pop('props') events.append(event) job = coroutine.spawn(read_events) coroutine.dispatch() @@ -95,14 +93,12 @@ class ServerCommandsTest(tests.Test): 'title': 'title_2', }) coroutine.dispatch() - ipc.delete(['context', guid]) coroutine.sleep(.5) job.kill() self.assertEqual([ {'guid': guid, 'document': 'context', 'event': 'create'}, {'guid': guid, 'document': 'context', 'event': 'update'}, - {'guid': guid, 'event': 'delete', 'document': 'context'}, ], events) diff --git a/tests/units/db/commands.py b/tests/units/db/commands.py index abb04c3..da859d5 100755 --- a/tests/units/db/commands.py +++ b/tests/units/db/commands.py @@ -161,7 +161,7 @@ class CommandsTest(tests.Test): self.assertRaises(CommandNotFound, self.call, cp, 'PROBE') self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', document='testdocument') self.assertRaises(NotFound, self.call, cp, 'PROBE', document='testdocument', guid='guid') - volume['testdocument'].create(guid='guid') + volume['testdocument'].create({'guid': 'guid'}) self.call(cp, 'PROBE', document='testdocument', guid='guid') self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', document='fakedocument', guid='guid') self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', cmd='command_1', document='testdocument', guid='guid') @@ -194,7 +194,7 @@ class CommandsTest(tests.Test): self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', document='testdocument') self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', document='testdocument', prop='prop') self.assertRaises(NotFound, self.call, cp, 'PROBE', document='testdocument', guid='guid', prop='prop') - volume['testdocument'].create(guid='guid') + volume['testdocument'].create({'guid': 'guid'}) self.call(cp, 'PROBE', document='testdocument', guid='guid', prop='prop') self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', document='fakedocument', guid='guid', prop='prop') self.assertRaises(CommandNotFound, self.call, cp, 'PROBE', cmd='command_1', document='testdocument', guid='guid', prop='prop') diff --git a/tests/units/db/document.py b/tests/units/db/document.py index a4f11ec..f3150fe 100755 --- a/tests/units/db/document.py +++ b/tests/units/db/document.py @@ -127,64 +127,6 @@ class DocumentTest(tests.Test): self.assertEqual(0, directory.find(0, 100, query='foo')[-1]) self.assertEqual(1, directory.find(0, 100, query='bar')[-1]) - def test_StoredProperty_Defaults(self): - - class Document(document.Document): - - @db.stored_property(default='default') - def w_default(self, value): - return value - - @db.stored_property() - def wo_default(self, value): - return value - - @db.indexed_property(slot=1, default='not_stored_default') - def not_stored_default(self, value): - return value - - directory = Directory(tests.tmpdir, Document, IndexWriter) - self.assertEqual('default', directory.metadata['w_default'].default) - self.assertEqual(None, directory.metadata['wo_default'].default) - self.assertEqual('not_stored_default', directory.metadata['not_stored_default'].default) - - guid = directory.create({'wo_default': 'wo_default'}) - - docs, total = directory.find(0, 100) - self.assertEqual(1, total) - self.assertEqual( - [('default', 'wo_default', 'not_stored_default')], - [(i.w_default, i.wo_default, i.not_stored_default) for i in docs]) - - self.assertRaises(RuntimeError, directory.create, {}) - - def test_properties_Blob(self): - - class Document(document.Document): - - @db.blob_property(mime_type='application/json') - def blob(self, value): - return value - - directory = Directory(tests.tmpdir, Document, IndexWriter) - - guid = directory.create({}) - blob_path = join(tests.tmpdir, guid[:2], guid, 'blob') - - self.assertEqual(db.PropertyMetadata(), directory.get(guid).blob) - - data = 'payload' - directory.set_blob(guid, 'blob', StringIO(data)) - self.assertEqual({ - 'seqno': 2, - 'mtime': int(os.stat(blob_path).st_mtime), - 'digest': hashlib.sha1(data).hexdigest(), - 'blob': join(tests.tmpdir, guid[:2], guid, 'blob.blob'), - 'mime_type': 'application/json', - }, - directory.get(guid).meta('blob')) - self.assertEqual(data, file(blob_path + '.blob').read()) - def test_update(self): class Document(document.Document): @@ -339,7 +281,7 @@ class DocumentTest(tests.Test): directory = Directory(tests.tmpdir, Document, IndexWriter) - guid = directory.create(guid='guid', prop='foo') + guid = directory.create({'guid': 'guid', 'prop': 'foo'}) self.assertEqual( [('guid', 'foo')], [(i.guid, i.prop) for i in directory.find(0, 1024)[0]]) @@ -353,17 +295,13 @@ class DocumentTest(tests.Test): class Document(document.Document): - @db.indexed_property(slot=1, default='') + @db.indexed_property(slot=1) def prop(self, value): return value - @db.blob_property() - def blob(self, value): - return value - directory = Directory(tests.tmpdir, Document, IndexWriter) - guid_1 = directory.create({}) + guid_1 = directory.create({'prop': 'value'}) seqno = directory.get(guid_1).get('seqno') self.assertEqual(1, seqno) self.assertEqual( @@ -373,7 +311,7 @@ class DocumentTest(tests.Test): json.load(file('%s/%s/prop' % (guid_1[:2], guid_1)))['seqno'], seqno) - guid_2 = directory.create({}) + guid_2 = directory.create({'prop': 'value'}) seqno = directory.get(guid_2).get('seqno') self.assertEqual(2, seqno) self.assertEqual( @@ -383,31 +321,15 @@ class DocumentTest(tests.Test): json.load(file('%s/%s/prop' % (guid_2[:2], guid_2)))['seqno'], seqno) - directory.set_blob(guid_1, 'blob', StringIO('blob')) - seqno = directory.get(guid_1).get('seqno') - self.assertEqual(3, seqno) - self.assertEqual( - json.load(file('%s/%s/guid' % (guid_1[:2], guid_1)))['seqno'], - 1) - self.assertEqual( - json.load(file('%s/%s/prop' % (guid_1[:2], guid_1)))['seqno'], - 1) - self.assertEqual( - json.load(file('%s/%s/blob' % (guid_1[:2], guid_1)))['seqno'], - seqno) - directory.update(guid_1, {'prop': 'new'}) seqno = directory.get(guid_1).get('seqno') - self.assertEqual(4, seqno) + self.assertEqual(3, seqno) self.assertEqual( json.load(file('%s/%s/guid' % (guid_1[:2], guid_1)))['seqno'], 1) self.assertEqual( json.load(file('%s/%s/prop' % (guid_1[:2], guid_1)))['seqno'], seqno) - self.assertEqual( - json.load(file('%s/%s/blob' % (guid_1[:2], guid_1)))['seqno'], - 3) def test_diff(self): @@ -423,17 +345,19 @@ class DocumentTest(tests.Test): directory = Directory(tests.tmpdir, Document, IndexWriter) - directory.create(guid='1', prop='1', ctime=1, mtime=1) - directory.set_blob('1', 'blob', StringIO('1')) + self.touch(('blob', '1')) + directory.create({'guid': '1', 'prop': '1', 'ctime': 1, 'mtime': 1}) + directory.update('1', {'blob': {'blob': 'blob'}}) for i in os.listdir('1/1'): os.utime('1/1/%s' % i, (1, 1)) - directory.create(guid='2', prop='2', ctime=2, mtime=2) - directory.set_blob('2', 'blob', StringIO('2')) + self.touch(('blob', '2')) + directory.create({'guid': '2', 'prop': '2', 'ctime': 2, 'mtime': 2}) + directory.update('2', {'blob': {'blob': 'blob'}}) for i in os.listdir('2/2'): os.utime('2/2/%s' % i, (2, 2)) - directory.create(guid='3', prop='3', ctime=3, mtime=3) + directory.create({'guid': '3', 'prop': '3', 'ctime': 3, 'mtime': 3}) for i in os.listdir('3/3'): os.utime('3/3/%s' % i, (3, 3)) @@ -446,8 +370,6 @@ class DocumentTest(tests.Test): 'mtime': {'value': 1, 'mtime': 1}, 'blob': { 'mtime': 1, - 'digest': hashlib.sha1('1').hexdigest(), - 'mime_type': 'application/octet-stream', 'blob': tests.tmpdir + '/1/1/blob.blob', }, }}, @@ -458,8 +380,6 @@ class DocumentTest(tests.Test): 'mtime': {'value': 2, 'mtime': 2}, 'blob': { 'mtime': 2, - 'digest': hashlib.sha1('2').hexdigest(), - 'mime_type': 'application/octet-stream', 'blob': tests.tmpdir + '/2/2/blob.blob', }, }}, @@ -482,8 +402,6 @@ class DocumentTest(tests.Test): 'mtime': {'value': 2, 'mtime': 2}, 'blob': { 'mtime': 2, - 'digest': hashlib.sha1('2').hexdigest(), - 'mime_type': 'application/octet-stream', 'blob': tests.tmpdir + '/2/2/blob.blob', }, }}, @@ -502,7 +420,7 @@ class DocumentTest(tests.Test): ], [i for i in diff(directory, [[6, 100]], out_seq)]) self.assertEqual([], out_seq) - directory.update(guid='2', prop='22') + directory.update('2', {'prop': '22'}) self.assertEqual([ {'guid': '2', 'diff': { 'prop': {'value': '22', 'mtime': int(os.stat('2/2/prop').st_mtime)}, @@ -521,7 +439,7 @@ class DocumentTest(tests.Test): directory = Directory('.', Document, IndexWriter) - directory.create(guid='guid', prop='1', ctime=1, mtime=1) + directory.create({'guid': 'guid', 'prop': '1', 'ctime': 1, 'mtime': 1}) self.utime('.', 1) out_seq = Sequence() @@ -535,7 +453,7 @@ class DocumentTest(tests.Test): [i for i in diff(directory, [[0, None]], out_seq)]) self.assertEqual([[1, 1]], out_seq) - directory.update(guid='guid', prop='2') + directory.update('guid', {'prop': '2'}) out_seq = Sequence() self.assertEqual([ ], @@ -552,10 +470,10 @@ class DocumentTest(tests.Test): directory = Directory(tests.tmpdir, Document, IndexWriter) - directory.create(guid='1', prop='1', ctime=1, mtime=1) - directory.create(guid='2', prop='2', ctime=2, mtime=2) - directory.create(guid='3', prop='3', ctime=3, mtime=3) - directory.update(guid='2', prop='2_') + directory.create({'guid': '1', 'prop': '1', 'ctime': 1, 'mtime': 1}) + directory.create({'guid': '2', 'prop': '2', 'ctime': 2, 'mtime': 2}) + directory.create({'guid': '3', 'prop': '3', 'ctime': 3, 'mtime': 3}) + directory.update('2', {'prop': '2_'}) self.utime('.', 0) out_seq = Sequence() @@ -586,8 +504,8 @@ class DocumentTest(tests.Test): directory = Directory(tests.tmpdir, Document, IndexWriter) - directory.create(guid='1', ctime=1, mtime=1) - directory.set_blob('1', 'blob', url=URL) + directory.create({'guid': '1', 'ctime': 1, 'mtime': 1}) + directory.update('1', {'blob': {'url': URL}}) self.utime('1/1', 1) out_seq = Sequence() @@ -598,7 +516,6 @@ class DocumentTest(tests.Test): 'mtime': {'value': 1, 'mtime': 1}, 'blob': { 'url': URL, - 'mime_type': 'application/octet-stream', 'mtime': 1, }, }}, @@ -616,8 +533,8 @@ class DocumentTest(tests.Test): directory = Directory(tests.tmpdir, Document, IndexWriter) - directory.create(guid='1', ctime=1, mtime=1, prop='1') - directory.create(guid='2', ctime=2, mtime=2, prop='2') + directory.create({'guid': '1', 'ctime': 1, 'mtime': 1, 'prop': '1'}) + directory.create({'guid': '2', 'ctime': 2, 'mtime': 2, 'prop': '2'}) for i in os.listdir('2/2'): os.utime('2/2/%s' % i, (2, 2)) @@ -643,10 +560,10 @@ class DocumentTest(tests.Test): directory = Directory(tests.tmpdir, Document, IndexWriter) - directory.create(guid='1', ctime=1, mtime=1, prop='0') + directory.create({'guid': '1', 'ctime': 1, 'mtime': 1, 'prop': '0'}) for i in os.listdir('1/1'): os.utime('1/1/%s' % i, (1, 1)) - directory.create(guid='2', ctime=2, mtime=2, prop='0') + directory.create({'guid': '2', 'ctime': 2, 'mtime': 2, 'prop': '0'}) for i in os.listdir('2/2'): os.utime('2/2/%s' % i, (2, 2)) @@ -676,17 +593,19 @@ class DocumentTest(tests.Test): directory1 = Directory('document1', Document, IndexWriter) - directory1.create(guid='1', prop='1', ctime=1, mtime=1) - directory1.set_blob('1', 'blob', StringIO('1')) + directory1.create({'guid': '1', 'prop': '1', 'ctime': 1, 'mtime': 1}) + self.touch(('blob', '1')) + directory1.update('1', {'blob': {'blob': 'blob'}}) for i in os.listdir('document1/1/1'): os.utime('document1/1/1/%s' % i, (1, 1)) - directory1.create(guid='2', prop='2', ctime=2, mtime=2) - directory1.set_blob('2', 'blob', StringIO('2')) + directory1.create({'guid': '2', 'prop': '2', 'ctime': 2, 'mtime': 2}) + self.touch(('blob', '2')) + directory1.update('2', {'blob': {'blob': 'blob'}}) for i in os.listdir('document1/2/2'): os.utime('document1/2/2/%s' % i, (2, 2)) - directory1.create(guid='3', prop='3', ctime=3, mtime=3) + directory1.create({'guid': '3', 'prop': '3', 'ctime': 3, 'mtime': 3}) for i in os.listdir('document1/3/3'): os.utime('document1/3/3/%s' % i, (3, 3)) @@ -737,13 +656,16 @@ class DocumentTest(tests.Test): directory1 = Directory('document1', Document, IndexWriter) directory2 = Directory('document2', Document, IndexWriter) - directory1.create(guid='guid', ctime=1, mtime=1) - directory1.set_blob('guid', 'blob', StringIO('1')) + directory1.create({'guid': 'guid', 'ctime': 1, 'mtime': 1}) + self.touch(('blob', '1')) + directory1.update('guid', {'blob': {'blob': 'blob'}}) for i in os.listdir('document1/gu/guid'): os.utime('document1/gu/guid/%s' % i, (1, 1)) - directory2.create(guid='guid', ctime=2, mtime=2) - directory2.set_blob('guid', 'blob', StringIO('2')) + directory2.create({'guid': 'guid', 'ctime': 2, 'mtime': 2}) + self.touch(('blob', '2')) + directory2.update('guid', {'blob': {'blob': 'blob'}}) + for i in os.listdir('document2/gu/guid'): os.utime('document2/gu/guid/%s' % i, (2, 2)) @@ -811,7 +733,7 @@ class DocumentTest(tests.Test): return value directory1 = Directory('document1', Document, IndexWriter) - directory1.create(guid='1', prop='1', ctime=1, mtime=1) + directory1.create({'guid': '1', 'prop': '1', 'ctime': 1, 'mtime': 1}) directory2 = Directory('document2', Document, IndexWriter) for patch in diff(directory1, [[0, None]], Sequence()): @@ -820,7 +742,7 @@ class DocumentTest(tests.Test): [(1, 1, '1', '1')], [(i['ctime'], i['mtime'], i['guid'], i['prop']) for i in directory2.find(0, 1024)[0]]) doc = directory2.get('1') - self.assertEqual(None, doc.get('seqno')) + self.assertEqual(0, doc.get('seqno')) self.assertEqual(0, doc.meta('guid')['seqno']) self.assertEqual(0, doc.meta('prop')['seqno']) @@ -836,7 +758,7 @@ class DocumentTest(tests.Test): self.assertEqual(1, doc.meta('prop')['seqno']) time.sleep(1) - directory1.update(guid='1', prop='2', ctime=2, mtime=2) + directory1.update('1', {'prop': '2', 'ctime': 2, 'mtime': 2}) for patch in diff(directory1, [[0, None]], Sequence()): directory3.merge(shift_seqno=False, **patch) @@ -849,7 +771,7 @@ class DocumentTest(tests.Test): self.assertEqual(1, doc.meta('prop')['seqno']) time.sleep(1) - directory1.update(guid='1', prop='3', ctime=3, mtime=3) + directory1.update('1', {'prop': '3', 'ctime': 3, 'mtime': 3}) for patch in diff(directory1, [[0, None]], Sequence()): directory3.merge(**patch) @@ -870,7 +792,7 @@ class DocumentTest(tests.Test): return {'url': 'http://foo/bar', 'mime_type': 'image/png'} directory1 = Directory('document1', Document, IndexWriter) - directory1.create(guid='guid', ctime=1, mtime=1) + directory1.create({'guid': 'guid', 'ctime': 1, 'mtime': 1}) for i in os.listdir('document1/gu/guid'): os.utime('document1/gu/guid/%s' % i, (1, 1)) @@ -892,11 +814,12 @@ class DocumentTest(tests.Test): return value directory = Directory('document', Document, IndexWriter) + self.touch(('blob', 'blob-1')) directory.merge('1', { 'guid': {'mtime': 1, 'value': '1'}, 'ctime': {'mtime': 2, 'value': 2}, 'mtime': {'mtime': 3, 'value': 3}, - 'blob': {'mtime': 4, 'blob': StringIO('blob-1')}, + 'blob': {'mtime': 4, 'blob': 'blob'}, }) self.assertEqual( @@ -911,8 +834,9 @@ class DocumentTest(tests.Test): self.assertEqual(4, doc.meta('blob')['mtime']) self.assertEqual('blob-1', file('document/1/1/blob.blob').read()) + self.touch(('blob', 'blob-2')) directory.merge('1', { - 'blob': {'mtime': 5, 'blob': StringIO('blob-2')}, + 'blob': {'mtime': 5, 'blob': 'blob'}, }) self.assertEqual(5, doc.meta('blob')['mtime']) diff --git a/tests/units/db/index.py b/tests/units/db/index.py index c51b436..7319765 100755 --- a/tests/units/db/index.py +++ b/tests/units/db/index.py @@ -29,12 +29,12 @@ class IndexTest(tests.Test): def test_Create(self): db = Index({'key': IndexedProperty('key', 1, 'K')}) - db.store('1', {'key': 'value_1'}, True) + db.store('1', {'key': 'value_1'}) self.assertEqual( ([{'guid': '1', 'key': 'value_1'}], 1), db._find(reply=['key'])) - db.store('2', {'key': 'value_2'}, True) + db.store('2', {'key': 'value_2'}) self.assertEqual( ([{'guid': '1', 'key': 'value_1'}, {'guid': '2', 'key': 'value_2'}], 2), @@ -46,12 +46,12 @@ class IndexTest(tests.Test): 'var_2': IndexedProperty('var_2', 2, 'B'), }) - db.store('1', {'var_1': 'value_1', 'var_2': 'value_2'}, True) + db.store('1', {'var_1': 'value_1', 'var_2': 'value_2'}) self.assertEqual( ([{'guid': '1', 'var_1': 'value_1', 'var_2': 'value_2'}], 1), db._find(reply=['var_1', 'var_2'])) - db.store('1', {'var_1': 'value_3', 'var_2': 'value_4'}, False) + db.store('1', {'var_1': 'value_3', 'var_2': 'value_4'}) self.assertEqual( ([{'guid': '1', 'var_1': 'value_3', 'var_2': 'value_4'}], 1), db._find(reply=['var_1', 'var_2'])) @@ -59,7 +59,7 @@ class IndexTest(tests.Test): def test_delete(self): db = Index({'key': IndexedProperty('key', 1, 'K')}) - db.store('1', {'key': 'value'}, True) + db.store('1', {'key': 'value'}) self.assertEqual( ([{'guid': '1', 'key': 'value'}], 1), db._find(reply=['key'])) @@ -72,7 +72,7 @@ class IndexTest(tests.Test): def test_IndexByReprcast(self): db = Index({'key': IndexedProperty('key', 1, 'K', reprcast=lambda x: "foo" + x)}) - db.store('1', {'key': 'bar'}, True) + db.store('1', {'key': 'bar'}) self.assertEqual( [{'guid': '1', 'key': 'foobar'}], @@ -96,7 +96,7 @@ class IndexTest(tests.Test): yield value db = Index({'key': IndexedProperty('key', 1, 'K', reprcast=iterate)}) - db.store('1', {'key': 'value'}, True) + db.store('1', {'key': 'value'}) self.assertEqual( [{'guid': '1'}], @@ -118,9 +118,9 @@ class IndexTest(tests.Test): 'var_3': IndexedProperty('var_3', 3, 'C', full_text=True), }) - db.store('1', {'var_1': '1', 'var_2': 'у', 'var_3': 'г'}, True) - db.store('2', {'var_1': '2', 'var_2': 'у', 'var_3': 'ю'}, True) - db.store('3', {'var_1': '3', 'var_2': 'б', 'var_3': 'ю'}, True) + db.store('1', {'var_1': '1', 'var_2': 'у', 'var_3': 'г'}) + db.store('2', {'var_1': '2', 'var_2': 'у', 'var_3': 'ю'}) + db.store('3', {'var_1': '3', 'var_2': 'б', 'var_3': 'ю'}) self.assertEqual( ([{'guid': '1', 'var_1': '1'}, @@ -147,7 +147,7 @@ class IndexTest(tests.Test): 'prop': IndexedProperty('prop', 1, 'P', full_text=True), }) - db.store('guid', {'prop': 'value'}, True) + db.store('guid', {'prop': 'value'}) self.assertEqual( [{'guid': 'guid', 'prop': 'value'}], @@ -164,8 +164,8 @@ class IndexTest(tests.Test): 'var_1': IndexedProperty('var_1', 1, 'A', typecast=bool), }) - db.store('1', {'var_1': True}, True) - db.store('2', {'var_1': False}, True) + db.store('1', {'var_1': True}) + db.store('2', {'var_1': False}) self.assertEqual( [{'guid': '1'}], @@ -181,9 +181,9 @@ class IndexTest(tests.Test): 'var_3': IndexedProperty('var_3', 3, 'C', full_text=True), }) - db.store('1', {'var_1': '1', 'var_2': 'у', 'var_3': 'г'}, True) - db.store('2', {'var_1': '2', 'var_2': 'у', 'var_3': 'ю'}, True) - db.store('3', {'var_1': '3', 'var_2': 'б', 'var_3': 'ю'}, True) + db.store('1', {'var_1': '1', 'var_2': 'у', 'var_3': 'г'}) + db.store('2', {'var_1': '2', 'var_2': 'у', 'var_3': 'ю'}) + db.store('3', {'var_1': '3', 'var_2': 'б', 'var_3': 'ю'}) self.assertEqual( ([{'guid': '1', 'var_1': '1'}, @@ -209,9 +209,9 @@ class IndexTest(tests.Test): 'var_3': IndexedProperty('var_3', 3, 'C', boolean=True, full_text=True), }) - db.store('1', {'var_1': '1', 'var_2': 'у', 'var_3': 'г'}, True) - db.store('2', {'var_1': '2', 'var_2': 'у', 'var_3': 'ю'}, True) - db.store('3', {'var_1': '3', 'var_2': 'б', 'var_3': 'ю'}, True) + db.store('1', {'var_1': '1', 'var_2': 'у', 'var_3': 'г'}) + db.store('2', {'var_1': '2', 'var_2': 'у', 'var_3': 'ю'}) + db.store('3', {'var_1': '3', 'var_2': 'б', 'var_3': 'ю'}) self.assertEqual( ([{'guid': '1', 'var_1': '1'}], 1), @@ -232,9 +232,9 @@ class IndexTest(tests.Test): 'var_3': IndexedProperty('var_3', 3, 'C', boolean=True, full_text=True), }) - db.store('1', {'var_1': '1', 'var_2': 'у', 'var_3': 'г'}, True) - db.store('2', {'var_1': '2', 'var_2': 'у', 'var_3': 'ю'}, True) - db.store('3', {'var_1': '3', 'var_2': 'б', 'var_3': 'ю'}, True) + db.store('1', {'var_1': '1', 'var_2': 'у', 'var_3': 'г'}) + db.store('2', {'var_1': '2', 'var_2': 'у', 'var_3': 'ю'}) + db.store('3', {'var_1': '3', 'var_2': 'б', 'var_3': 'ю'}) self.assertEqual( ([{'guid': '1', 'var_1': '1'}], 1), @@ -251,9 +251,9 @@ class IndexTest(tests.Test): def test_find_ExactQuery(self): db = Index({'key': IndexedProperty('key', 1, 'K', full_text=True)}) - db.store('1', {'key': 'фу'}, True) - db.store('2', {'key': 'фу бар'}, True) - db.store('3', {'key': 'фу бар тест'}, True) + db.store('1', {'key': 'фу'}) + db.store('2', {'key': 'фу бар'}) + db.store('3', {'key': 'фу бар тест'}) self.assertEqual( ([{'guid': '1', 'key': u'фу'}, {'guid': '2', 'key': u'фу бар'}, {'guid': '3', 'key': u'фу бар тест'}], 3), @@ -277,8 +277,8 @@ class IndexTest(tests.Test): db = Index({term: IndexedProperty(term, 1, 'T', full_text=True)}) - db.store('1', {term: 'test'}, True) - db.store('2', {term: 'test fail'}, True) + db.store('1', {term: 'test'}) + db.store('2', {term: 'test fail'}) self.assertEqual( ([{'guid': '1'}], 1), @@ -287,9 +287,9 @@ class IndexTest(tests.Test): def test_find_ReturnPortions(self): db = Index({'key': IndexedProperty('key', 1, 'K')}) - db.store('1', {'key': '1'}, True) - db.store('2', {'key': '2'}, True) - db.store('3', {'key': '3'}, True) + db.store('1', {'key': '1'}) + db.store('2', {'key': '2'}) + db.store('3', {'key': '3'}) self.assertEqual( ([{'guid': '1', 'key': '1'}], 3), @@ -310,9 +310,9 @@ class IndexTest(tests.Test): 'var_2': IndexedProperty('var_2', 2, 'B'), }) - db.store('1', {'var_1': '1', 'var_2': '3'}, True) - db.store('2', {'var_1': '2', 'var_2': '2'}, True) - db.store('3', {'var_1': '3', 'var_2': '1'}, True) + db.store('1', {'var_1': '1', 'var_2': '3'}) + db.store('2', {'var_1': '2', 'var_2': '2'}) + db.store('3', {'var_1': '3', 'var_2': '1'}) self.assertEqual( ([{'guid': '1'}, {'guid': '2'}, {'guid': '3'}], 3), @@ -342,9 +342,9 @@ class IndexTest(tests.Test): 'var_4': IndexedProperty('var_4', 4, 'D'), }) - db.store('1', {'var_1': '1', 'var_2': '1', 'var_3': '3', 'var_4': 0}, True) - db.store('2', {'var_1': '2', 'var_2': '1', 'var_3': '4', 'var_4': 0}, True) - db.store('3', {'var_1': '3', 'var_2': '2', 'var_3': '4', 'var_4': 0}, True) + db.store('1', {'var_1': '1', 'var_2': '1', 'var_3': '3', 'var_4': 0}) + db.store('2', {'var_1': '2', 'var_2': '1', 'var_3': '4', 'var_4': 0}) + db.store('3', {'var_1': '3', 'var_2': '2', 'var_3': '4', 'var_4': 0}) self.assertEqual( [{'guid': '1', 'var_1': '1'}, {'guid': '3', 'var_1': '3'}], @@ -363,8 +363,8 @@ class IndexTest(tests.Test): db = Index({ 'prop': IndexedProperty('prop', prefix='B', typecast=[1, 2], full_text=True), }) - db.store('1', {'prop': [1, 2]}, True) - db.store('2', {'prop': [2, 3]}, True) + db.store('1', {'prop': [1, 2]}) + db.store('2', {'prop': [2, 3]}) self.assertEqual( [{'guid': '1'}], db._find(prop=1, reply=['guid'])[0]) @@ -382,8 +382,8 @@ class IndexTest(tests.Test): db = Index({ 'prop': IndexedProperty('prop', prefix='B', typecast=[], full_text=True), }) - db.store('1', {'prop': ['a', 'b']}, True) - db.store('2', {'prop': ['b', 'c']}, True) + db.store('1', {'prop': ['a', 'b']}) + db.store('2', {'prop': ['b', 'c']}) self.assertEqual( [{'guid': '1'}], db._find(prop='a', reply=['guid'])[0]) @@ -405,13 +405,13 @@ class IndexTest(tests.Test): post_stored = [] deleted = [] - db.store('1', {}, True, + db.store('1', {}, lambda *args: pre_stored.append(args), lambda *args: post_stored.append(args)) self.assertEqual(1, len(pre_stored)) self.assertEqual(1, len(post_stored)) - db.store('1', {}, False, + db.store('1', {}, lambda *args: pre_stored.append(args), lambda *args: post_stored.append(args)) self.assertEqual(2, len(pre_stored)) @@ -424,7 +424,7 @@ class IndexTest(tests.Test): # No index at start; checkpoint didn't happen db = Index({}) self.assertEqual(0, db.mtime) - db.store('1', {}, True) + db.store('1', {}) db.commit() db.close() @@ -437,7 +437,7 @@ class IndexTest(tests.Test): os.utime('index/mtime', (1, 1)) db = Index({}) self.assertEqual(1, db.mtime) - db.store('3', {}, True) + db.store('3', {}) db.commit() self.assertNotEqual(1, db.mtime) db.close() @@ -445,9 +445,9 @@ class IndexTest(tests.Test): def test_find_OrderByGUIDAllTime(self): db = Index({'prop': IndexedProperty('prop', 1, 'P')}) - db.store('3', {'prop': '1'}, True) - db.store('2', {'prop': '1'}, True) - db.store('1', {'prop': '3'}, True) + db.store('3', {'prop': '1'}) + db.store('2', {'prop': '1'}) + db.store('1', {'prop': '3'}) self.assertEqual( ([{'guid': '1', 'prop': '3'}, {'guid': '2', 'prop': '1'}, {'guid': '3', 'prop': '1'}], 3), @@ -466,8 +466,8 @@ class IndexTest(tests.Test): db = Index({term: IndexedProperty(term, 1, 'T', full_text=True)}) - db.store('1', {term: 'test'}, True) - db.store('2', {term: 'test fail'}, True) + db.store('1', {term: 'test'}) + db.store('2', {term: 'test fail'}) self.assertEqual( ([{'guid': '1'}], 1), @@ -476,9 +476,9 @@ class IndexTest(tests.Test): def test_find_WithListProps(self): db = Index({'prop': IndexedProperty('prop', None, 'A', full_text=True, typecast=[])}) - db.store('1', {'prop': ('a', )}, True) - db.store('2', {'prop': ('a', 'aa')}, True) - db.store('3', {'prop': ('aa', 'aaa')}, True) + db.store('1', {'prop': ('a', )}) + db.store('2', {'prop': ('a', 'aa')}) + db.store('3', {'prop': ('aa', 'aaa')}) self.assertEqual( ([{'guid': '1'}, {'guid': '2'}], 2), @@ -498,11 +498,11 @@ class IndexTest(tests.Test): db = Index({}, lambda: commits.append(True)) coroutine.dispatch() env.index_flush_threshold.value = 1 - db.store('1', {}, True) + db.store('1', {}) coroutine.dispatch() - db.store('2', {}, True) + db.store('2', {}) coroutine.dispatch() - db.store('3', {}, True) + db.store('3', {}) coroutine.dispatch() self.assertEqual(3, len(commits)) db.close() @@ -511,15 +511,15 @@ class IndexTest(tests.Test): db = Index({}, lambda: commits.append(True)) coroutine.dispatch() env.index_flush_threshold.value = 2 - db.store('4', {}, True) + db.store('4', {}) coroutine.dispatch() - db.store('5', {}, True) + db.store('5', {}) coroutine.dispatch() - db.store('6', {}, True) + db.store('6', {}) coroutine.dispatch() - db.store('7', {}, True) + db.store('7', {}) coroutine.dispatch() - db.store('8', {}, True) + db.store('8', {}) coroutine.dispatch() self.assertEqual(2, len(commits)) db.close() @@ -533,20 +533,20 @@ class IndexTest(tests.Test): db = Index({}, lambda: commits.append(True)) coroutine.dispatch() - db.store('1', {}, True) + db.store('1', {}) coroutine.dispatch() self.assertEqual(0, len(commits)) - db.store('2', {}, True) + db.store('2', {}) coroutine.dispatch() self.assertEqual(0, len(commits)) coroutine.sleep(1.5) self.assertEqual(1, len(commits)) - db.store('1', {}, True) + db.store('1', {}) coroutine.dispatch() self.assertEqual(1, len(commits)) - db.store('2', {}, True) + db.store('2', {}) coroutine.dispatch() self.assertEqual(1, len(commits)) @@ -561,7 +561,7 @@ class IndexTest(tests.Test): commits = [] db = Index({}, lambda: commits.append(True)) - db.store('1', {}, True) + db.store('1', {}) coroutine.dispatch() self.assertEqual(1, len(commits)) @@ -571,10 +571,10 @@ class IndexTest(tests.Test): db = Index({'prop': IndexedProperty('prop', 1, 'A', localized=True)}) - db.store('0', {'prop': {'foo': '5'}}, True) - db.store('1', {'prop': {current_lang: '4', 'default_lang': '1', 'foo': '3'}}, True) - db.store('2', {'prop': {'default_lang': '2', 'foo': '2'}}, True) - db.store('3', {'prop': {current_lang: '3', 'foo': '6'}}, True) + db.store('0', {'prop': {'foo': '5'}}) + db.store('1', {'prop': {current_lang: '4', 'default_lang': '1', 'foo': '3'}}) + db.store('2', {'prop': {'default_lang': '2', 'foo': '2'}}) + db.store('3', {'prop': {current_lang: '3', 'foo': '6'}}) self.assertEqual([ {'guid': '1'}, @@ -595,9 +595,9 @@ class IndexTest(tests.Test): def test_find_MultipleFilter(self): db = Index({'prop': IndexedProperty('prop', 1, 'A')}) - db.store('1', {'prop': 'a'}, True) - db.store('2', {'prop': 'b'}, True) - db.store('3', {'prop': 'c'}, True) + db.store('1', {'prop': 'a'}) + db.store('2', {'prop': 'b'}) + db.store('3', {'prop': 'c'}) self.assertEqual( sorted([ @@ -637,9 +637,9 @@ class IndexTest(tests.Test): def test_find_NotFilter(self): db = Index({'prop': IndexedProperty('prop', 1, 'A')}) - db.store('1', {'prop': 'a'}, True) - db.store('2', {'prop': 'b'}, True) - db.store('3', {'prop': 'c'}, True) + db.store('1', {'prop': 'a'}) + db.store('2', {'prop': 'b'}) + db.store('3', {'prop': 'c'}) self.assertEqual( sorted([ @@ -676,9 +676,9 @@ class IndexTest(tests.Test): def test_find_AndNotFilter(self): db = Index({'prop': IndexedProperty('prop', 1, 'A')}) - db.store('1', {'prop': 'a'}, True) - db.store('2', {'prop': 'b'}, True) - db.store('3', {'prop': 'c'}, True) + db.store('1', {'prop': 'a'}) + db.store('2', {'prop': 'b'}) + db.store('3', {'prop': 'c'}) self.assertEqual( sorted([ diff --git a/tests/units/db/storage.py b/tests/units/db/storage.py index e75b72e..149ed03 100755 --- a/tests/units/db/storage.py +++ b/tests/units/db/storage.py @@ -56,86 +56,6 @@ class StorageTest(tests.Test): }, storage.get('guid').get('prop')) - def test_Record_set_blob_ByStream(self): - storage = self.storage([BlobProperty('prop')]) - - record = storage.get('guid1') - data = '!' * BUFFER_SIZE * 2 - record.set_blob('prop', StringIO(data)) - self.assertEqual({ - 'blob': tests.tmpdir + '/gu/guid1/prop.blob', - 'mtime': int(os.stat('gu/guid1/prop').st_mtime), - 'digest': hashlib.sha1(data).hexdigest(), - }, - record.get('prop')) - self.assertEqual(data, file('gu/guid1/prop.blob').read()) - - record = storage.get('guid2') - record.set_blob('prop', StringIO('12345'), 1) - self.assertEqual({ - 'blob': tests.tmpdir + '/gu/guid2/prop.blob', - 'mtime': int(os.stat('gu/guid2/prop').st_mtime), - 'digest': hashlib.sha1('1').hexdigest(), - }, - record.get('prop')) - self.assertEqual('1', file('gu/guid2/prop.blob').read()) - - def test_Record_set_blob_ByPath(self): - storage = self.storage([BlobProperty('prop')]) - - record = storage.get('guid1') - self.touch(('file', 'data')) - record.set_blob('prop', tests.tmpdir + '/file') - self.assertEqual({ - 'blob': tests.tmpdir + '/gu/guid1/prop.blob', - 'mtime': int(os.stat('gu/guid1/prop').st_mtime), - 'digest': hashlib.sha1('data').hexdigest(), - }, - record.get('prop')) - self.assertEqual('data', file('gu/guid1/prop.blob').read()) - - record = storage.get('guid2') - self.touch(('directory/1', '1')) - self.touch(('directory/2/3', '3')) - self.touch(('directory/2/4/5', '5')) - record.set_blob('prop', tests.tmpdir + '/directory') - self.assertEqual({ - 'blob': tests.tmpdir + '/gu/guid2/prop.blob', - 'mtime': int(os.stat('gu/guid2/prop').st_mtime), - 'digest': hashlib.sha1( - '1' '1' - '2/3' '3' - '2/4/5' '5' - ).hexdigest(), - }, - record.get('prop')) - util.assert_call('diff -r directory gu/guid2/prop.blob', shell=True) - - def test_Record_set_blob_ByUrl(self): - storage = self.storage([BlobProperty('prop')]) - record = storage.get('guid1') - - record.set_blob('prop', url='http://sugarlabs.org') - self.assertEqual({ - 'url': 'http://sugarlabs.org', - 'mtime': int(os.stat('gu/guid1/prop').st_mtime), - }, - record.get('prop')) - assert not exists('gu/guid1/prop.blob') - - def test_Record_set_blob_ByValue(self): - storage = self.storage([BlobProperty('prop')]) - record = storage.get('guid') - - record.set_blob('prop', '/foo/bar') - self.assertEqual({ - 'blob': tests.tmpdir + '/gu/guid/prop.blob', - 'mtime': int(os.stat('gu/guid/prop').st_mtime), - 'digest': hashlib.sha1('/foo/bar').hexdigest(), - }, - record.get('prop')) - self.assertEqual('/foo/bar', file('gu/guid/prop.blob').read()) - def test_delete(self): storage = self.storage([StoredProperty('prop')]) @@ -143,8 +63,6 @@ class StorageTest(tests.Test): storage.delete('absent') record = storage.get('guid') - self.touch(('directory/1/2/3', '3')) - record.set_blob('prop', 'directory') record.set('prop', value='value') assert exists('gu/guid') storage.delete('guid') diff --git a/tests/units/db/volume.py b/tests/units/db/volume.py index 5ffd262..5c250b9 100755 --- a/tests/units/db/volume.py +++ b/tests/units/db/volume.py @@ -17,7 +17,7 @@ from __init__ import tests from sugar_network import db, toolkit from sugar_network.db import env from sugar_network.db.volume import VolumeCommands -from sugar_network.toolkit import coroutine, http +from sugar_network.toolkit import coroutine, http, util class VolumeTest(tests.Test): @@ -26,6 +26,29 @@ class VolumeTest(tests.Test): tests.Test.setUp(self) self.response = db.Response() + def test_PostDefaults(self): + + class Document(db.Document): + + @db.stored_property(default='default') + def w_default(self, value): + return value + + @db.stored_property() + def wo_default(self, value): + return value + + @db.indexed_property(slot=1, default='not_stored_default') + def not_stored_default(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [Document]) + guid = self.call('POST', document='document', content={}) + + self.assertEqual('default', self.call('GET', document='document', guid=guid, prop='w_default')) + self.assertEqual(None, self.call('GET', document='document', guid=guid, prop='wo_default')) + self.assertEqual('not_stored_default', self.call('GET', document='document', guid=guid, prop='not_stored_default')) + def test_Populate(self): self.touch( ('document/1/1/guid', '{"value": "1"}'), @@ -76,7 +99,7 @@ class VolumeTest(tests.Test): return value self.volume = db.Volume(tests.tmpdir, [TestDocument]) - self.volume['testdocument'].create(guid='guid') + self.volume['testdocument'].create({'guid': 'guid'}) self.assertEqual({ 'total': 1, @@ -139,8 +162,6 @@ class VolumeTest(tests.Test): self.volume = db.Volume(tests.tmpdir, [TestDocument]) guid = self.call('POST', document='testdocument', content={}) - self.assertRaises(RuntimeError, self.call, 'PUT', document='testdocument', guid=guid, prop='blob', content={'path': '/'}) - self.call('PUT', document='testdocument', guid=guid, prop='blob', content='blob1') self.assertEqual('blob1', file(self.call('GET', document='testdocument', guid=guid, prop='blob')['blob']).read()) @@ -150,6 +171,66 @@ class VolumeTest(tests.Test): self.call('PUT', document='testdocument', guid=guid, prop='blob', content=None) self.assertRaises(http.NotFound, self.call, 'GET', document='testdocument', guid=guid, prop='blob') + def test_SetBLOBsByMeta(self): + + class TestDocument(db.Document): + + @db.blob_property(mime_type='default') + def blob(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument]) + guid = self.call('POST', document='testdocument', content={}) + + self.assertRaises(RuntimeError, self.call, 'PUT', document='testdocument', guid=guid, prop='blob', + content={}, content_type='application/json') + self.assertRaises(http.NotFound, self.call, 'GET', document='testdocument', guid=guid, prop='blob') + + self.touch('file') + self.assertRaises(RuntimeError, self.call, 'PUT', document='testdocument', guid=guid, prop='blob', + content={'blob': 'file'}, content_type='application/json') + self.assertRaises(http.NotFound, self.call, 'GET', document='testdocument', guid=guid, prop='blob') + + self.call('PUT', document='testdocument', guid=guid, prop='blob', + content={'url': 'foo', 'bar': 'probe'}, content_type='application/json') + blob = self.call('GET', document='testdocument', guid=guid, prop='blob') + self.assertEqual('foo', blob['url']) + assert 'bar' not in blob + + def test_RemoveBLOBs(self): + + class TestDocument(db.Document): + + @db.blob_property(mime_type='default') + def blob(self, value): + return value + + self.volume = db.Volume(tests.tmpdir, [TestDocument]) + guid = self.call('POST', document='testdocument', content={'blob': 'blob'}) + + self.assertEqual('blob', file(self.call('GET', document='testdocument', guid=guid, prop='blob')['blob']).read()) + + self.call('PUT', document='testdocument', guid=guid, prop='blob') + self.assertRaises(http.NotFound, self.call, 'GET', document='testdocument', guid=guid, prop='blob') + + def test_RemoveTempBLOBFilesOnFails(self): + + class TestDocument(db.Document): + + @db.blob_property(mime_type='default') + def blob(self, value): + return value + + @blob.setter + def blob(self, value): + raise RuntimeError() + + self.volume = db.Volume(tests.tmpdir, [TestDocument]) + guid = self.call('POST', document='testdocument', content={}) + + self.assertRaises(RuntimeError, self.call, 'PUT', document='testdocument', guid=guid, prop='blob', content='probe') + self.assertEqual(0, len(os.listdir('tmp'))) + def test_SetBLOBsWithMimeType(self): class TestDocument(db.Document): @@ -221,7 +302,7 @@ class VolumeTest(tests.Test): ], self.call('GET', document='testdocument', reply=['blob'], static_prefix='http://127.0.0.1')['result']) - self.volume['testdocument'].set_blob(guid, 'blob', 'file') + self.call('PUT', document='testdocument', guid=guid, prop='blob', content='file') self.assertEqual('file', file(self.call('GET', document='testdocument', guid=guid, prop='blob')['blob']).read()) self.assertEqual( {'blob': 'http://127.0.0.1/testdocument/%s/blob' % guid}, @@ -231,7 +312,8 @@ class VolumeTest(tests.Test): ], self.call('GET', document='testdocument', reply=['blob'], static_prefix='http://127.0.0.1')['result']) - self.volume['testdocument'].set_blob(guid, 'blob', db.PropertyMetadata(url='http://foo')) + self.call('PUT', document='testdocument', guid=guid, prop='blob', content={'url': 'http://foo'}, + content_type='application/json') self.assertEqual('http://foo', self.call('GET', document='testdocument', guid=guid, prop='blob')['url']) self.assertEqual( {'blob': 'http://foo'}, @@ -469,7 +551,7 @@ class VolumeTest(tests.Test): 'http://sugarlabs.org', self.call('GET', document='testdocument', guid=guid, prop='blob')['url']) - def test_before_create(self): + def test_on_create(self): class TestDocument(db.Document): @@ -488,13 +570,13 @@ class VolumeTest(tests.Test): assert self.volume['testdocument'].get(guid)['ctime'] in range(ts - 1, ts + 1) assert self.volume['testdocument'].get(guid)['mtime'] in range(ts - 1, ts + 1) - def test_before_create_Override(self): + def test_on_create_Override(self): class Commands(VolumeCommands): - def before_create(self, request, props): + def on_create(self, request, props, event): props['prop'] = 'overriden' - VolumeCommands.before_create(self, request, props) + VolumeCommands.on_create(self, request, props, event) class TestDocument(db.Document): @@ -519,7 +601,7 @@ class VolumeTest(tests.Test): cp.call(request, db.Response()) self.assertEqual('bar', volume['testdocument'].get(guid)['prop']) - def test_before_update(self): + def test_on_update(self): class TestDocument(db.Document): @@ -540,13 +622,13 @@ class VolumeTest(tests.Test): self.call(method='PUT', document='testdocument', guid=guid, content={'prop': 'probe'}) assert self.volume['testdocument'].get(guid)['mtime'] - prev_mtime >= 1 - def test_before_update_Override(self): + def test_on_update_Override(self): class Commands(VolumeCommands): - def before_update(self, request, props): + def on_update(self, request, props, event): props['prop'] = 'overriden' - VolumeCommands.before_update(self, request, props) + VolumeCommands.on_update(self, request, props, event) class TestDocument(db.Document): @@ -601,13 +683,13 @@ class VolumeTest(tests.Test): assert not exists('seqno') self.assertEqual(0, volume.seqno.value) - volume['document1'].create(guid='1') + volume['document1'].create({'guid': '1'}) self.assertEqual(1, volume['document1'].get('1')['seqno']) - volume['document2'].create(guid='1') + volume['document2'].create({'guid': '1'}) self.assertEqual(2, volume['document2'].get('1')['seqno']) - volume['document1'].create(guid='2') + volume['document1'].create({'guid': '2'}) self.assertEqual(3, volume['document1'].get('2')['seqno']) - volume['document2'].create(guid='2') + volume['document2'].create({'guid': '2'}) self.assertEqual(4, volume['document2'].get('2')['seqno']) self.assertEqual(4, volume.seqno.value) @@ -650,52 +732,28 @@ class VolumeTest(tests.Test): volume.connect(lambda event: events.append(event)) volume.populate() + mtime = int(os.stat('document1/index/mtime').st_mtime) self.assertEqual([ - {'event': 'commit', 'document': 'document1'}, - {'event': 'populate', 'document': 'document1'}, + {'event': 'commit', 'document': 'document1', 'mtime': mtime}, + {'event': 'populate', 'document': 'document1', 'mtime': mtime}, ], events) del events[:] - volume['document1'].create(guid='guid1') - volume['document2'].create(guid='guid2') + volume['document1'].create({'guid': 'guid1'}) + volume['document2'].create({'guid': 'guid2'}) self.assertEqual([ - {'event': 'create', 'document': 'document1', 'guid': 'guid1', 'props': { - 'ctime': 0, - 'mtime': 0, - 'seqno': 0, - 'prop': '', - 'guid': 'guid1', - }}, - {'event': 'create', 'document': 'document2', 'guid': 'guid2', 'props': { - 'ctime': 0, - 'mtime': 0, - 'seqno': 0, - 'prop': '', - 'guid': 'guid2', - }}, + {'event': 'create', 'document': 'document1', 'guid': 'guid1'}, + {'event': 'create', 'document': 'document2', 'guid': 'guid2'}, ], events) del events[:] - volume['document1'].update('guid1', prop='foo') - volume['document2'].update('guid2', prop='bar') + volume['document1'].update('guid1', {'prop': 'foo'}) + volume['document2'].update('guid2', {'prop': 'bar'}) self.assertEqual([ - {'event': 'update', 'document': 'document1', 'guid': 'guid1', 'props': { - 'prop': 'foo', - }}, - {'event': 'update', 'document': 'document2', 'guid': 'guid2', 'props': { - 'prop': 'bar', - }}, - ], - events) - del events[:] - - volume['document2'].set_blob('guid2', 'blob', StringIO('blob')) - self.assertEqual([ - {'event': 'update', 'document': 'document2', 'guid': 'guid2', 'props': { - 'seqno': 5, - }}, + {'event': 'update', 'document': 'document1', 'guid': 'guid1'}, + {'event': 'update', 'document': 'document2', 'guid': 'guid2'}, ], events) del events[:] @@ -708,11 +766,13 @@ class VolumeTest(tests.Test): del events[:] volume['document1'].commit() + mtime1 = int(os.stat('document1/index/mtime').st_mtime) volume['document2'].commit() + mtime2 = int(os.stat('document2/index/mtime').st_mtime) self.assertEqual([ - {'event': 'commit', 'document': 'document1'}, - {'event': 'commit', 'document': 'document2'}, + {'event': 'commit', 'document': 'document1', 'mtime': mtime1}, + {'event': 'commit', 'document': 'document2', 'mtime': mtime2}, ], events) @@ -815,7 +875,7 @@ class VolumeTest(tests.Test): @blob1.setter def blob1(self, value): - return db.PropertyMetadata(url=value) + return db.PropertyMetadata(url=file(value['blob']).read()) @db.blob_property() def blob2(self, meta): @@ -823,7 +883,10 @@ class VolumeTest(tests.Test): @blob2.setter def blob2(self, value): - return ' %s ' % value + with util.NamedTemporaryFile(delete=False) as f: + f.write(' %s ' % file(value['blob']).read()) + value['blob'] = f.name + return value self.volume = db.Volume(tests.tmpdir, [TestDocument]) guid = self.call('POST', document='testdocument', content={}) @@ -839,8 +902,8 @@ class VolumeTest(tests.Test): self.assertEqual('_3', self.call('GET', document='testdocument', guid=guid, prop='prop')) self.assertRaises(http.NotFound, self.call, 'GET', document='testdocument', guid=guid, prop='blob1') - self.call('PUT', document='testdocument', guid=guid, prop='blob1', content='blob2') - self.assertEqual('blob2', self.call('GET', document='testdocument', guid=guid, prop='blob1')['url']) + self.call('PUT', document='testdocument', guid=guid, prop='blob1', content='blob_url') + self.assertEqual('blob_url', self.call('GET', document='testdocument', guid=guid, prop='blob1')['url']) guid = self.call('POST', document='testdocument', content={'blob2': 'foo'}) self.assertEqual(' foo ', file(self.call('GET', document='testdocument', guid=guid, prop='blob2')['blob']).read()) @@ -887,11 +950,15 @@ class VolumeTest(tests.Test): @blob.setter def blob(self, value): - if '!' not in value: + blob = file(value['blob']).read() + if '!' not in blob: meta = self.meta('blob') if meta: - value = file(meta['blob']).read() + value - coroutine.spawn(self.post, value) + blob = file(meta['blob']).read() + blob + with util.NamedTemporaryFile(delete=False) as f: + f.write(blob) + value['blob'] = f.name + coroutine.spawn(self.post, blob) return value def post(self, value): diff --git a/tests/units/node/master.py b/tests/units/node/master.py index 4e78aaf..242e159 100755 --- a/tests/units/node/master.py +++ b/tests/units/node/master.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # sugar-lint: disable +import os + from __init__ import tests from sugar_network.node import obs @@ -162,7 +164,7 @@ class MasterTest(tests.Test): 'description': 'description', }) coroutine.sleep(.5) - self.assertEqual(0, len(events)) + self.assertEqual([], events) ipc.put(['context', guid, 'aliases'], { 'Gentoo': { @@ -171,8 +173,10 @@ class MasterTest(tests.Test): }, }) coroutine.sleep(.5) - self.assertEqual(1, len(events)) - assert 'mtime' in events[0]['props'] + self.assertEqual([ + {'event': 'populate', 'document': 'implementation', 'mtime': int(os.stat('master/implementation/index/mtime').st_mtime)}, + ], + events) self.assertEqual({ 'Gentoo-2.1': {'status': 'success', 'binary': ['bin'], 'devel': ['devel']}, }, @@ -200,9 +204,10 @@ class MasterTest(tests.Test): ipc.put(['context', guid, 'dependencies'], ['foo']) coroutine.sleep(.1) - self.assertEqual(1, len(events)) - assert 'mtime' in events[0]['props'] - del events[:] + self.assertEqual([ + {'event': 'populate', 'document': 'implementation', 'mtime': int(os.stat('master/implementation/index/mtime').st_mtime)}, + ], + events) if __name__ == '__main__': diff --git a/tests/units/node/node.py b/tests/units/node/node.py index e28f6f3..9e7f0c9 100755 --- a/tests/units/node/node.py +++ b/tests/units/node/node.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # sugar-lint: disable +import os import time import json from email.utils import formatdate, parsedate @@ -10,7 +11,7 @@ from __init__ import tests from sugar_network import db, node from sugar_network.client import Client -from sugar_network.toolkit import http +from sugar_network.toolkit import http, coroutine from sugar_network.toolkit.rrd import Rrd from sugar_network.node import stats_user, stats_node, obs from sugar_network.node.commands import NodeCommands @@ -152,11 +153,40 @@ class NodeTest(tests.Test): call(cp, method='GET', document='context', guid=guid, reply=['guid', 'title', 'layer'])) self.assertEqual(['public'], volume['context'].get(guid)['layer']) + events = [] + volume.connect(lambda event: events.append(event)) call(cp, method='DELETE', document='context', guid=guid, principal='principal') + coroutine.dispatch() - assert exists(guid_path) self.assertRaises(http.NotFound, call, cp, method='GET', document='context', guid=guid, reply=['guid', 'title']) self.assertEqual(['deleted'], volume['context'].get(guid)['layer']) + self.assertEqual([ + {'event': 'delete', 'document': 'context', 'guid': guid}, + {'event': 'commit', 'document': 'context', 'mtime': int(os.stat('db/context/index/mtime').st_mtime)}, + ], + events) + + def test_SimulateDeleteEvents(self): + volume = Volume('db') + cp = NodeCommands('guid', volume) + + guid = call(cp, method='POST', document='context', principal='principal', content={ + 'type': 'activity', + 'title': 'title', + 'summary': 'summary', + 'description': 'description', + }) + + events = [] + volume.connect(lambda event: events.append(event)) + call(cp, method='PUT', document='context', guid=guid, principal='principal', content={'layer': ['deleted']}) + coroutine.dispatch() + + self.assertEqual([ + {'event': 'delete', 'document': 'context', 'guid': guid}, + {'event': 'commit', 'document': 'context', 'mtime': int(os.stat('db/context/index/mtime').st_mtime)}, + ], + events) def test_RegisterUser(self): cp = NodeCommands('guid', Volume('db', [User])) @@ -287,7 +317,7 @@ class NodeTest(tests.Test): call(cp, method='GET', document='context', guid=guid) self.assertNotEqual([], call(cp, method='GET', document='context')['result']) - volume['context'].update(guid, layer=['deleted']) + volume['context'].update(guid, {'layer': ['deleted']}) self.assertRaises(http.NotFound, call, cp, method='GET', document='context', guid=guid) self.assertEqual([], call(cp, method='GET', document='context')['result']) diff --git a/tests/units/node/stats_node.py b/tests/units/node/stats_node.py index 5d91ef8..607a380 100755 --- a/tests/units/node/stats_node.py +++ b/tests/units/node/stats_node.py @@ -56,13 +56,13 @@ class StatsTest(tests.Test): self.assertEqual(0, stats._stats['solution'].total) self.assertEqual(0, stats._stats['artifact'].total) - volume['user'].create(guid='user', name='user', color='', pubkey='') - volume['context'].create(guid='context', type='activity', title='', summary='', description='') - volume['review'].create(guid='review', context='context', title='', content='', rating=5) - volume['feedback'].create(guid='feedback', context='context', type='idea', title='', content='') - volume['feedback'].create(guid='feedback2', context='context', type='idea', title='', content='', solution='solution') - volume['solution'].create(guid='solution', context='context', feedback='feedback', content='') - volume['artifact'].create(guid='artifact', type='instance', context='context', title='', description='') + volume['user'].create({'guid': 'user', 'name': 'user', 'color': '', 'pubkey': ''}) + volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) + volume['review'].create({'guid': 'review', 'context': 'context', 'title': '', 'content': '', 'rating': 5}) + volume['feedback'].create({'guid': 'feedback', 'context': 'context', 'type': 'idea', 'title': '', 'content': ''}) + volume['feedback'].create({'guid': 'feedback2', 'context': 'context', 'type': 'idea', 'title': '', 'content': '', 'solution': 'solution'}) + volume['solution'].create({'guid': 'solution', 'context': 'context', 'feedback': 'feedback', 'content': ''}) + volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''}) stats = Sniffer(volume) self.assertEqual(1, stats._stats['user'].total) @@ -136,7 +136,7 @@ class StatsTest(tests.Test): def test_FeedbackSolutions(self): volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) stats = Sniffer(volume) - volume['feedback'].create(guid='guid', context='context', type='idea', title='', content='') + volume['feedback'].create({'guid': 'guid', 'context': 'context', 'type': 'idea', 'title': '', 'content': ''}) request = db.Request(method='PUT', document='feedback', guid='guid') request.principal = 'user' @@ -164,9 +164,9 @@ class StatsTest(tests.Test): def test_Comments(self): volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) stats = Sniffer(volume) - volume['solution'].create(guid='solution', context='context', feedback='feedback', content='') - volume['feedback'].create(guid='feedback', context='context', type='idea', title='', content='') - volume['review'].create(guid='review', context='context', title='', content='', rating=5) + volume['solution'].create({'guid': 'solution', 'context': 'context', 'feedback': 'feedback', 'content': ''}) + volume['feedback'].create({'guid': 'feedback', 'context': 'context', 'type': 'idea', 'title': '', 'content': ''}) + volume['review'].create({'guid': 'review', 'context': 'context', 'title': '', 'content': '', 'rating': 5}) request = db.Request(method='POST', document='comment') request.principal = 'user' @@ -189,8 +189,8 @@ class StatsTest(tests.Test): def test_Reviewes(self): volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) stats = Sniffer(volume) - volume['context'].create(guid='context', type='activity', title='', summary='', description='') - volume['artifact'].create(guid='artifact', type='instance', context='context', title='', description='') + volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) + volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''}) request = db.Request(method='POST', document='review') request.principal = 'user' @@ -216,8 +216,8 @@ class StatsTest(tests.Test): def test_ContextDownloaded(self): volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact, Implementation]) stats = Sniffer(volume) - volume['context'].create(guid='context', type='activity', title='', summary='', description='') - volume['implementation'].create(guid='implementation', context='context', license='GPLv3', version='1', date=0, stability='stable', notes='') + volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) + volume['implementation'].create({'guid': 'implementation', 'context': 'context', 'license': 'GPLv3', 'version': '1', 'date': 0, 'stability': 'stable', 'notes': ''}) request = db.Request(method='GET', document='implementation', guid='implementation', prop='fake') request.principal = 'user' @@ -232,7 +232,7 @@ class StatsTest(tests.Test): def test_ContextReleased(self): volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact, Implementation]) stats = Sniffer(volume) - volume['context'].create(guid='context', type='activity', title='', summary='', description='') + volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) request = db.Request(method='POST', document='implementation') request.principal = 'user' @@ -243,7 +243,7 @@ class StatsTest(tests.Test): def test_ContextFailed(self): volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact, Implementation]) stats = Sniffer(volume) - volume['context'].create(guid='context', type='activity', title='', summary='', description='') + volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) request = db.Request(method='POST', document='report') request.principal = 'user' @@ -269,7 +269,7 @@ class StatsTest(tests.Test): ['1', '2'], stats._stats['context'].active.keys()) - volume['artifact'].create(guid='artifact', type='instance', context='3', title='', description='') + volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': '3', 'title': '', 'description': ''}) request = db.Request(method='GET', document='review', artifact='artifact') request.principal = 'user' stats.log(request) @@ -277,7 +277,7 @@ class StatsTest(tests.Test): ['1', '2', '3'], sorted(stats._stats['context'].active.keys())) - volume['feedback'].create(guid='feedback', context='4', type='idea', title='', content='') + volume['feedback'].create({'guid': 'feedback', 'context': '4', 'type': 'idea', 'title': '', 'content': ''}) request = db.Request(method='GET', document='solution', feedback='feedback') request.principal = 'user' stats.log(request) @@ -300,7 +300,7 @@ class StatsTest(tests.Test): ['1', '2', '3', '4', '5', '6'], sorted(stats._stats['context'].active.keys())) - volume['solution'].create(guid='solution', context='7', feedback='feedback', content='') + volume['solution'].create({'guid': 'solution', 'context': '7', 'feedback': 'feedback', 'content': ''}) request = db.Request(method='POST', document='comment') request.principal = 'user' request.content = {'solution': 'solution'} @@ -336,7 +336,7 @@ class StatsTest(tests.Test): def test_ArtifactDownloaded(self): volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) stats = Sniffer(volume) - volume['artifact'].create(guid='artifact', type='instance', context='context', title='', description='') + volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''}) request = db.Request(method='GET', document='artifact', guid='artifact', prop='fake') request.principal = 'user' @@ -353,12 +353,12 @@ class StatsTest(tests.Test): def test_Commit(self): stats_node_step.value = 1 volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact]) - volume['user'].create(guid='user', name='user', color='', pubkey='') - volume['context'].create(guid='context', type='activity', title='', summary='', description='') - volume['review'].create(guid='review', context='context', title='', content='', rating=5) - volume['feedback'].create(guid='feedback', context='context', type='idea', title='', content='') - volume['solution'].create(guid='solution', context='context', feedback='feedback', content='') - volume['artifact'].create(guid='artifact', type='instance', context='context', title='', description='') + volume['user'].create({'guid': 'user', 'name': 'user', 'color': '', 'pubkey': ''}) + volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) + volume['review'].create({'guid': 'review', 'context': 'context', 'title': '', 'content': '', 'rating': 5}) + volume['feedback'].create({'guid': 'feedback', 'context': 'context', 'type': 'idea', 'title': '', 'content': ''}) + volume['solution'].create({'guid': 'solution', 'context': 'context', 'feedback': 'feedback', 'content': ''}) + volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''}) stats = Sniffer(volume) request = db.Request(method='GET', document='user', guid='user') @@ -480,9 +480,9 @@ class StatsTest(tests.Test): stats_node_step.value = 1 volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact, Implementation]) - volume['context'].create(guid='context', type='activity', title='', summary='', description='') - volume['implementation'].create(guid='implementation', context='context', license='GPLv3', version='1', date=0, stability='stable', notes='') - volume['artifact'].create(guid='artifact', type='instance', context='context', title='', description='') + volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) + volume['implementation'].create({'guid': 'implementation', 'context': 'context', 'license': 'GPLv3', 'version': '1', 'date': 0, 'stability': 'stable', 'notes': ''}) + volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''}) self.assertEqual([0, 0], volume['context'].get('context')['reviews']) self.assertEqual(0, volume['context'].get('context')['rating']) @@ -526,8 +526,8 @@ class StatsTest(tests.Test): stats_node_step.value = 1 volume = Volume('local', [User, Context, Review, Feedback, Solution, Artifact, Implementation]) - volume['context'].create(guid='context', type='activity', title='', summary='', description='') - volume['artifact'].create(guid='artifact', type='instance', context='context', title='', description='') + volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) + volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''}) self.assertEqual([0, 0], volume['artifact'].get('artifact')['reviews']) self.assertEqual(0, volume['artifact'].get('artifact')['rating']) diff --git a/tests/units/node/sync_master.py b/tests/units/node/sync_master.py index b8e83f5..e137f6e 100755 --- a/tests/units/node/sync_master.py +++ b/tests/units/node/sync_master.py @@ -313,8 +313,8 @@ class SyncMasterTest(tests.Test): response.get('set-cookie')) def test_pull(self): - self.volume['document'].create(guid='1', prop='1', ctime=1, mtime=1) - self.volume['document'].create(guid='2', prop='2', ctime=2, mtime=2) + self.volume['document'].create({'guid': '1', 'prop': '1', 'ctime': 1, 'mtime': 1}) + self.volume['document'].create({'guid': '2', 'prop': '2', 'ctime': 2, 'mtime': 2}) self.utime('master', 0) self.touch(('sync/1', 'file1')) self.touch(('sync/2', 'file2')) @@ -539,8 +539,8 @@ class SyncMasterTest(tests.Test): response.get('set-cookie')) def test_pull_ExcludeSentCookies(self): - self.volume['document'].create(guid='1', prop='1', ctime=1, mtime=1) - self.volume['document'].create(guid='2', prop='2', ctime=2, mtime=2) + self.volume['document'].create({'guid': '1', 'prop': '1', 'ctime': 1, 'mtime': 1}) + self.volume['document'].create({'guid': '2', 'prop': '2', 'ctime': 2, 'mtime': 2}) self.utime('master', 0) request = Request() @@ -587,8 +587,8 @@ class SyncMasterTest(tests.Test): response.get('set-cookie')) def test_pull_DoNotExcludeSentCookiesForMultipleNodes(self): - self.volume['document'].create(guid='1', prop='1', ctime=1, mtime=1) - self.volume['document'].create(guid='2', prop='2', ctime=2, mtime=2) + self.volume['document'].create({'guid': '1', 'prop': '1', 'ctime': 1, 'mtime': 1}) + self.volume['document'].create({'guid': '2', 'prop': '2', 'ctime': 2, 'mtime': 2}) self.utime('master', 0) request = Request() diff --git a/tests/units/node/sync_offline.py b/tests/units/node/sync_offline.py index fa3b947..752ff93 100755 --- a/tests/units/node/sync_offline.py +++ b/tests/units/node/sync_offline.py @@ -70,8 +70,8 @@ class SyncOfflineTest(tests.Test): cp = SlaveCommands('node/key', volume) stats_user.stats_user.value = True - volume['document'].create(guid='1', prop='value1', ctime=1, mtime=1) - volume['document'].create(guid='2', prop='value2', ctime=2, mtime=2) + volume['document'].create({'guid': '1', 'prop': 'value1', 'ctime': 1, 'mtime': 1}) + volume['document'].create({'guid': '2', 'prop': 'value2', 'ctime': 2, 'mtime': 2}) self.utime('node', 0) ts = int(time.time()) @@ -122,8 +122,8 @@ class SyncOfflineTest(tests.Test): cp = SlaveCommands('node/key', volume) stats_user.stats_user.value = True - volume['document'].create(guid='1', prop=payload, ctime=1, mtime=1) - volume['document'].create(guid='2', prop=payload, ctime=2, mtime=2) + volume['document'].create({'guid': '1', 'prop': payload, 'ctime': 1, 'mtime': 1}) + volume['document'].create({'guid': '2', 'prop': payload, 'ctime': 2, 'mtime': 2}) self.utime('node', 0) ts = int(time.time()) diff --git a/tests/units/node/volume.py b/tests/units/node/volume.py index c93fc21..db1cdd4 100755 --- a/tests/units/node/volume.py +++ b/tests/units/node/volume.py @@ -2,6 +2,7 @@ # sugar-lint: disable import os +import time import urllib2 import hashlib from cStringIO import StringIO @@ -11,6 +12,7 @@ from __init__ import tests from sugar_network import db from sugar_network.node.volume import diff, merge from sugar_network.node.stats_node import stats_node_step, Sniffer +from sugar_network.node.commands import NodeCommands from sugar_network.resources.user import User from sugar_network.resources.volume import Volume, Resource from sugar_network.resources.review import Review @@ -23,6 +25,10 @@ from sugar_network.toolkit import util class VolumeTest(tests.Test): + def setUp(self): + tests.Test.setUp(self) + self.override(time, 'time', lambda: 0) + def test_diff(self): class Document(db.Document): @@ -32,27 +38,27 @@ class VolumeTest(tests.Test): return value volume = Volume('db', [Document]) - volume['document'].create(guid='1', seqno=1, prop='a') - for i in os.listdir('db/document/1/1'): - os.utime('db/document/1/1/%s' % i, (1, 1)) - volume['document'].create(guid='2', seqno=2, prop='b') - for i in os.listdir('db/document/2/2'): - os.utime('db/document/2/2/%s' % i, (2, 2)) + cp = NodeCommands('guid', volume) + + guid1 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'a'}) + self.utime('db/document/%s/%s' % (guid1[:2], guid1), 1) + guid2 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'b'}) + self.utime('db/document/%s/%s' % (guid2[:2], guid2), 2) in_seq = util.Sequence([[1, None]]) self.assertEqual([ {'document': 'document'}, - {'guid': '1', + {'guid': guid1, 'diff': { - 'guid': {'value': '1', 'mtime': 1}, + 'guid': {'value': guid1, 'mtime': 1}, 'mtime': {'value': 0, 'mtime': 1}, 'ctime': {'value': 0, 'mtime': 1}, 'prop': {'value': 'a', 'mtime': 1}, }, }, - {'guid': '2', + {'guid': guid2, 'diff': { - 'guid': {'value': '2', 'mtime': 2}, + 'guid': {'value': guid2, 'mtime': 2}, 'mtime': {'value': 0, 'mtime': 2}, 'ctime': {'value': 0, 'mtime': 2}, 'prop': {'value': 'b', 'mtime': 2}, @@ -72,17 +78,17 @@ class VolumeTest(tests.Test): return value volume = Volume('db', [Document]) - volume['document'].create(guid='1', seqno=1, prop='a') - for i in os.listdir('db/document/1/1'): - os.utime('db/document/1/1/%s' % i, (1, 1)) - volume['document'].create(guid='2', seqno=2, prop='b') - for i in os.listdir('db/document/2/2'): - os.utime('db/document/2/2/%s' % i, (2, 2)) + cp = NodeCommands('guid', volume) + + guid1 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'a'}) + self.utime('db/document/%s/%s' % (guid1[:2], guid1), 1) + guid2 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'b'}) + self.utime('db/document/%s/%s' % (guid2[:2], guid2), 2) in_seq = util.Sequence([[1, None]]) patch = diff(volume, in_seq) self.assertEqual({'document': 'document'}, next(patch)) - self.assertEqual('1', next(patch)['guid']) + self.assertEqual(guid1, next(patch)['guid']) self.assertEqual({'commit': []}, patch.throw(StopIteration())) try: next(patch) @@ -92,8 +98,8 @@ class VolumeTest(tests.Test): patch = diff(volume, in_seq) self.assertEqual({'document': 'document'}, next(patch)) - self.assertEqual('1', next(patch)['guid']) - self.assertEqual('2', next(patch)['guid']) + self.assertEqual(guid1, next(patch)['guid']) + self.assertEqual(guid2, next(patch)['guid']) self.assertEqual({'commit': [[1, 1]]}, patch.throw(StopIteration())) try: next(patch) @@ -110,22 +116,25 @@ class VolumeTest(tests.Test): return value volume = Volume('db', [Document]) - volume['document'].create(guid='1', seqno=1, prop='a') - for i in os.listdir('db/document/1/1'): - os.utime('db/document/1/1/%s' % i, (1, 1)) - volume['document'].create(guid='3', seqno=3, prop='c') - for i in os.listdir('db/document/3/3'): - os.utime('db/document/3/3/%s' % i, (3, 3)) - volume['document'].create(guid='5', seqno=5, prop='f') - for i in os.listdir('db/document/5/5'): - os.utime('db/document/5/5/%s' % i, (5, 5)) + cp = NodeCommands('guid', volume) + + guid1 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'a'}) + self.utime('db/document/%s/%s' % (guid1[:2], guid1), 1) + guid2 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'b'}) + volume['document'].delete(guid2) + guid3 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'c'}) + self.utime('db/document/%s/%s' % (guid3[:2], guid3), 2) + guid4 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'd'}) + volume['document'].delete(guid4) + guid5 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'f'}) + self.utime('db/document/%s/%s' % (guid5[:2], guid5), 2) in_seq = util.Sequence([[1, None]]) patch = diff(volume, in_seq) self.assertEqual({'document': 'document'}, patch.send(None)) - self.assertEqual('1', patch.send(None)['guid']) - self.assertEqual('3', patch.send(None)['guid']) - self.assertEqual('5', patch.send(None)['guid']) + self.assertEqual(guid1, patch.send(None)['guid']) + self.assertEqual(guid3, patch.send(None)['guid']) + self.assertEqual(guid5, patch.send(None)['guid']) self.assertEqual({'commit': [[1, 1], [3, 3]]}, patch.throw(StopIteration())) try: patch.send(None) @@ -135,9 +144,9 @@ class VolumeTest(tests.Test): patch = diff(volume, in_seq) self.assertEqual({'document': 'document'}, patch.send(None)) - self.assertEqual('1', patch.send(None)['guid']) - self.assertEqual('3', patch.send(None)['guid']) - self.assertEqual('5', patch.send(None)['guid']) + self.assertEqual(guid1, patch.send(None)['guid']) + self.assertEqual(guid3, patch.send(None)['guid']) + self.assertEqual(guid5, patch.send(None)['guid']) self.assertEqual({'commit': [[1, 5]]}, patch.send(None)) try: patch.send(None) @@ -154,18 +163,24 @@ class VolumeTest(tests.Test): return value volume = Volume('db', [Document]) - volume['document'].create(guid='3', seqno=3, prop='c') - for i in os.listdir('db/document/3/3'): - os.utime('db/document/3/3/%s' % i, (3, 3)) - volume['document'].create(guid='5', seqno=5, prop='f') - for i in os.listdir('db/document/5/5'): - os.utime('db/document/5/5/%s' % i, (5, 5)) + cp = NodeCommands('guid', volume) + + guid1 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'a'}) + volume['document'].delete(guid1) + guid2 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'b'}) + volume['document'].delete(guid2) + guid3 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'c'}) + self.utime('db/document/%s/%s' % (guid3[:2], guid3), 2) + guid4 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'd'}) + volume['document'].delete(guid4) + guid5 = call(cp, method='POST', document='document', principal='principal', content={'prop': 'f'}) + self.utime('db/document/%s/%s' % (guid5[:2], guid5), 2) in_seq = util.Sequence([[1, None]]) patch = diff(volume, in_seq, util.Sequence([[1, 1]])) self.assertEqual({'document': 'document'}, patch.send(None)) - self.assertEqual('3', patch.send(None)['guid']) - self.assertEqual('5', patch.send(None)['guid']) + self.assertEqual(guid3, patch.send(None)['guid']) + self.assertEqual(guid5, patch.send(None)['guid']) self.assertEqual({'commit': [[1, 1], [3, 3], [5, 5]]}, patch.send(None)) try: patch.send(None) @@ -185,24 +200,23 @@ class VolumeTest(tests.Test): pass volume = Volume('db', [Document1, Document2, Document3]) - volume['document1'].create(guid='3', seqno=3) - for i in os.listdir('db/document1/3/3'): - os.utime('db/document1/3/3/%s' % i, (3, 3)) - volume['document2'].create(guid='2', seqno=2) - for i in os.listdir('db/document2/2/2'): - os.utime('db/document2/2/2/%s' % i, (2, 2)) - volume['document3'].create(guid='1', seqno=1) - for i in os.listdir('db/document3/1/1'): - os.utime('db/document3/1/1/%s' % i, (1, 1)) + cp = NodeCommands('guid', volume) + + guid3 = call(cp, method='POST', document='document1', principal='principal', content={}) + self.utime('db/document/%s/%s' % (guid3[:2], guid3), 3) + guid2 = call(cp, method='POST', document='document2', principal='principal', content={}) + self.utime('db/document/%s/%s' % (guid2[:2], guid2), 2) + guid1 = call(cp, method='POST', document='document3', principal='principal', content={}) + self.utime('db/document/%s/%s' % (guid1[:2], guid1), 1) in_seq = util.Sequence([[1, None]]) patch = diff(volume, in_seq) self.assertEqual({'document': 'document1'}, patch.send(None)) - self.assertEqual('3', patch.send(None)['guid']) + self.assertEqual(guid3, patch.send(None)['guid']) self.assertEqual({'document': 'document2'}, patch.send(None)) - self.assertEqual('2', patch.send(None)['guid']) + self.assertEqual(guid2, patch.send(None)['guid']) self.assertEqual({'document': 'document3'}, patch.send(None)) - self.assertEqual('1', patch.send(None)['guid']) + self.assertEqual(guid1, patch.send(None)['guid']) self.assertEqual({'commit': [[1, 3]]}, patch.send(None)) try: patch.send(None) @@ -267,7 +281,7 @@ class VolumeTest(tests.Test): self.touch(('db/seqno', '100')) volume = Volume('db', [Document]) - volume['document'].create(guid='1', prop='1', ctime=1, mtime=1) + volume['document'].create({'guid': '1', 'prop': '1', 'ctime': 1, 'mtime': 1}) for i in os.listdir('db/document/1/1'): os.utime('db/document/1/1/%s' % i, (2, 2)) @@ -333,21 +347,24 @@ class VolumeTest(tests.Test): def test_merge_UpdateReviewStats(self): stats_node_step.value = 1 volume = Volume('db', [User, Context, Review, Feedback, Solution, Artifact]) + cp = NodeCommands('guid', volume) stats = Sniffer(volume) - volume['context'].create( - guid='context', - implement='guid', - type='package', - title='title', - summary='summary', - description='description') - volume['artifact'].create( - guid='artifact', - type='instance', - context='context', - title='', - description='') + context = call(cp, method='POST', document='context', principal='principal', content={ + 'guid': 'context', + 'implement': 'guid', + 'type': 'package', + 'title': 'title', + 'summary': 'summary', + 'description': 'description', + }) + artifact = call(cp, method='POST', document='artifact', principal='principal', content={ + 'guid': 'artifact', + 'type': 'instance', + 'context': context, + 'title': '', + 'description': '', + }) records = [ {'document': 'review'}, @@ -355,15 +372,15 @@ class VolumeTest(tests.Test): 'guid': {'value': '1', 'mtime': 1.0}, 'ctime': {'value': 1, 'mtime': 1.0}, 'mtime': {'value': 1, 'mtime': 1.0}, - 'context': {'value': 'context', 'mtime': 1.0}, - 'artifact': {'value': 'artifact', 'mtime': 4.0}, + 'context': {'value': context, 'mtime': 1.0}, + 'artifact': {'value': artifact, 'mtime': 4.0}, 'rating': {'value': 1, 'mtime': 1.0}, }}, {'guid': '2', 'diff': { 'guid': {'value': '2', 'mtime': 2.0}, 'ctime': {'value': 2, 'mtime': 2.0}, 'mtime': {'value': 2, 'mtime': 2.0}, - 'context': {'value': 'context', 'mtime': 2.0}, + 'context': {'value': context, 'mtime': 2.0}, 'rating': {'value': 2, 'mtime': 2.0}, }}, {'commit': [[1, 2]]}, @@ -371,22 +388,24 @@ class VolumeTest(tests.Test): merge(volume, records, node_stats=stats) stats.commit() - self.assertEqual(1, volume['artifact'].get('artifact')['rating']) - self.assertEqual([1, 1], volume['artifact'].get('artifact')['reviews']) - self.assertEqual(2, volume['context'].get('context')['rating']) - self.assertEqual([1, 2], volume['context'].get('context')['reviews']) + self.assertEqual(1, volume['artifact'].get(artifact)['rating']) + self.assertEqual([1, 1], volume['artifact'].get(artifact)['reviews']) + self.assertEqual(2, volume['context'].get(context)['rating']) + self.assertEqual([1, 2], volume['context'].get(context)['reviews']) def test_diff_Blobs(self): - class Document(db.Document): + class Document(Resource): @db.blob_property() def prop(self, value): return value - volume = Volume('db', [Document]) - volume['document'].create(guid='1', seqno=1) - volume['document'].set_blob('1', 'prop', 'payload') + volume = Volume('db', [User, Document]) + cp = NodeCommands('guid', volume) + + guid = call(cp, method='POST', document='document', principal='principal', content={}) + call(cp, method='PUT', document='document', guid=guid, principal='principal', content={'prop': 'payload'}) self.utime('db', 0) patch = diff(volume, util.Sequence([[1, None]])) @@ -396,7 +415,7 @@ class VolumeTest(tests.Test): record = next(patch) self.assertEqual('payload', ''.join([i for i in record.pop('blob')])) self.assertEqual( - {'guid': '1', 'blob_size': len('payload'), 'diff': { + {'guid': guid, 'blob_size': len('payload'), 'diff': { 'prop': { 'digest': hashlib.sha1('payload').hexdigest(), 'mime_type': 'application/octet-stream', @@ -405,42 +424,55 @@ class VolumeTest(tests.Test): }}, record) self.assertEqual( - {'guid': '1', 'diff': { - 'guid': {'value': '1', 'mtime': 0}, + {'guid': guid, 'diff': { + 'guid': {'value': guid, 'mtime': 0}, + 'author': {'value': {'principal': {'order': 0, 'role': 2}}, 'mtime': 0}, + 'layer': {'mtime': 0, 'value': ['public']}, + 'tags': {'mtime': 0, 'value': []}, 'mtime': {'value': 0, 'mtime': 0}, 'ctime': {'value': 0, 'mtime': 0}, }}, next(patch)) self.assertEqual( - {'commit': [[1, 1]]}, + {'document': 'user'}, next(patch)) + self.assertEqual( + {'commit': [[1, 2]]}, + next(patch)) + self.assertRaises(StopIteration, next, patch) def test_diff_BlobUrls(self): url = 'http://src.sugarlabs.org/robots.txt' blob = urllib2.urlopen(url).read() - class Document(db.Document): + class Document(Resource): @db.blob_property() def prop(self, value): return value - volume = Volume('db', [Document]) - volume['document'].create(guid='1', seqno=1) - volume['document'].set_blob('1', 'prop', url=url) + volume = Volume('db', [User, Document]) + cp = NodeCommands('guid', volume) + + guid = call(cp, method='POST', document='document', principal='principal', content={}) + call(cp, method='PUT', document='document', guid=guid, principal='principal', content={'prop': {'url': url}}) self.utime('db', 1) self.assertEqual([ {'document': 'document'}, - {'guid': '1', + {'guid': guid, 'diff': { - 'guid': {'value': '1', 'mtime': 1}, + 'guid': {'value': guid, 'mtime': 1}, + 'author': {'value': {'principal': {'order': 0, 'role': 2}}, 'mtime': 1}, + 'layer': {'mtime': 1, 'value': ['public']}, + 'tags': {'mtime': 1, 'value': []}, 'mtime': {'value': 0, 'mtime': 1}, 'ctime': {'value': 0, 'mtime': 1}, - 'prop': {'url': url, 'mime_type': 'application/octet-stream', 'mtime': 1}, + 'prop': {'url': url, 'mtime': 1}, }, }, - {'commit': [[1, 1]]}, + {'document': 'user'}, + {'commit': [[1, 2]]}, ], [i for i in diff(volume, util.Sequence([[1, None]]))]) @@ -451,76 +483,94 @@ class VolumeTest(tests.Test): record = next(patch) self.assertEqual(blob, ''.join([i for i in record.pop('blob')])) self.assertEqual( - {'guid': '1', 'blob_size': len(blob), 'diff': { - 'prop': { - 'mime_type': 'application/octet-stream', - 'mtime': 1, - }, - }}, + {'guid': guid, 'blob_size': len(blob), 'diff': {'prop': {'mtime': 1}}}, record) self.assertEqual( - {'guid': '1', 'diff': { - 'guid': {'value': '1', 'mtime': 1}, + {'guid': guid, 'diff': { + 'guid': {'value': guid, 'mtime': 1}, + 'author': {'value': {'principal': {'order': 0, 'role': 2}}, 'mtime': 1}, + 'layer': {'mtime': 1, 'value': ['public']}, + 'tags': {'mtime': 1, 'value': []}, 'mtime': {'value': 0, 'mtime': 1}, 'ctime': {'value': 0, 'mtime': 1}, }}, next(patch)) self.assertEqual( - {'commit': [[1, 1]]}, + {'document': 'user'}, next(patch)) + self.assertEqual( + {'commit': [[1, 2]]}, + next(patch)) + self.assertRaises(StopIteration, next, patch) def test_diff_SkipBrokenBlobUrls(self): - class Document(db.Document): + class Document(Resource): @db.blob_property() def prop(self, value): return value - volume = Volume('db', [Document]) - volume['document'].create(guid='1') - volume['document'].set_blob('1', 'prop', url='http://foo/bar') - volume['document'].create(guid='2') + volume = Volume('db', [User, Document]) + cp = NodeCommands('guid', volume) + + guid1 = call(cp, method='POST', document='document', principal='principal', content={}) + call(cp, method='PUT', document='document', guid=guid1, principal='principal', content={'prop': {'url': 'http://foo/bar'}}) + guid2 = call(cp, method='POST', document='document', principal='principal', content={}) self.utime('db', 1) self.assertEqual([ {'document': 'document'}, - {'guid': '1', + {'guid': guid1, 'diff': { - 'guid': {'value': '1', 'mtime': 1}, + 'guid': {'value': guid1, 'mtime': 1}, + 'author': {'value': {'principal': {'order': 0, 'role': 2}}, 'mtime': 1}, + 'layer': {'mtime': 1, 'value': ['public']}, + 'tags': {'mtime': 1, 'value': []}, 'mtime': {'value': 0, 'mtime': 1}, 'ctime': {'value': 0, 'mtime': 1}, - 'prop': {'url': 'http://foo/bar', 'mime_type': 'application/octet-stream', 'mtime': 1}, + 'prop': {'url': 'http://foo/bar', 'mtime': 1}, }, }, - {'guid': '2', + {'guid': guid2, 'diff': { - 'guid': {'value': '2', 'mtime': 1}, + 'guid': {'value': guid2, 'mtime': 1}, + 'author': {'value': {'principal': {'order': 0, 'role': 2}}, 'mtime': 1}, + 'layer': {'mtime': 1, 'value': ['public']}, + 'tags': {'mtime': 1, 'value': []}, 'mtime': {'value': 0, 'mtime': 1}, 'ctime': {'value': 0, 'mtime': 1}, }, }, + {'document': 'user'}, {'commit': [[1, 3]]}, ], [i for i in diff(volume, util.Sequence([[1, None]]), fetch_blobs=False)]) self.assertEqual([ {'document': 'document'}, - {'guid': '1', + {'guid': guid1, 'diff': { - 'guid': {'value': '1', 'mtime': 1}, + 'guid': {'value': guid1, 'mtime': 1}, + 'author': {'value': {'principal': {'order': 0, 'role': 2}}, 'mtime': 1}, + 'layer': {'mtime': 1, 'value': ['public']}, + 'tags': {'mtime': 1, 'value': []}, 'mtime': {'value': 0, 'mtime': 1}, 'ctime': {'value': 0, 'mtime': 1}, }, }, - {'guid': '2', + {'guid': guid2, 'diff': { - 'guid': {'value': '2', 'mtime': 1}, + 'guid': {'value': guid2, 'mtime': 1}, + 'author': {'value': {'principal': {'order': 0, 'role': 2}}, 'mtime': 1}, + 'layer': {'mtime': 1, 'value': ['public']}, + 'tags': {'mtime': 1, 'value': []}, 'mtime': {'value': 0, 'mtime': 1}, 'ctime': {'value': 0, 'mtime': 1}, }, }, - {'commit': [[1, 1], [3, 3]]}, + {'document': 'user'}, + {'commit': [[1, 3]]}, ], [i for i in diff(volume, util.Sequence([[1, None]]), fetch_blobs=True)]) @@ -571,15 +621,15 @@ class VolumeTest(tests.Test): pass volume = Volume('db', [Context, Implementation, Review]) - volume['context'].create(guid='0', ctime=1, mtime=1, layer=['layer0', 'common']) - volume['context'].create(guid='1', ctime=1, mtime=1, layer='layer1') - volume['implementation'].create(guid='2', ctime=2, mtime=2, layer='layer2') - volume['review'].create(guid='3', ctime=3, mtime=3, layer='layer3') - - volume['context'].update(guid='0', tags='0') - volume['context'].update(guid='1', tags='1') - volume['implementation'].update(guid='2', tags='2') - volume['review'].update(guid='3', tags='3') + volume['context'].create({'guid': '0', 'ctime': 1, 'mtime': 1, 'layer': ['layer0', 'common']}) + volume['context'].create({'guid': '1', 'ctime': 1, 'mtime': 1, 'layer': 'layer1'}) + volume['implementation'].create({'guid': '2', 'ctime': 2, 'mtime': 2, 'layer': 'layer2'}) + volume['review'].create({'guid': '3', 'ctime': 3, 'mtime': 3, 'layer': 'layer3'}) + + volume['context'].update('0', {'tags': '0'}) + volume['context'].update('1', {'tags': '1'}) + volume['implementation'].update('2', {'tags': '2'}) + volume['review'].update('3', {'tags': '3'}) self.utime('db', 0) self.assertEqual(sorted([ @@ -627,5 +677,13 @@ class VolumeTest(tests.Test): sorted([i for i in diff(volume, util.Sequence([[5, None]]), layer='foo')])) +def call(cp, principal=None, content=None, **kwargs): + request = db.Request(**kwargs) + request.principal = principal + request.content = content + request.environ = {'HTTP_HOST': '127.0.0.1'} + return cp.call(request, db.Response()) + + if __name__ == '__main__': tests.main() diff --git a/tests/units/resources/comment.py b/tests/units/resources/comment.py index 13e24cd..9c89517 100755 --- a/tests/units/resources/comment.py +++ b/tests/units/resources/comment.py @@ -11,6 +11,7 @@ from sugar_network.resources.feedback import Feedback from sugar_network.resources.solution import Solution from sugar_network.resources.comment import Comment from sugar_network.resources.implementation import Implementation +from sugar_network.toolkit import http class CommentTest(tests.Test): @@ -19,7 +20,9 @@ class CommentTest(tests.Test): volume = self.start_master([User, Context, Review, Feedback, Solution, Comment, Implementation]) client = Client() - self.assertRaises(RuntimeError, client.post, ['comment'], {'message': ''}) + self.assertRaises(http.NotFound, client.post, ['comment'], {'message': '', 'review': 'absent'}) + self.assertRaises(http.NotFound, client.post, ['comment'], {'message': '', 'feedback': 'absent'}) + self.assertRaises(http.NotFound, client.post, ['comment'], {'message': '', 'solution': 'absent'}) context = client.post(['context'], { 'type': 'package', diff --git a/tests/units/resources/implementation.py b/tests/units/resources/implementation.py index 46c9768..3b711f6 100755 --- a/tests/units/resources/implementation.py +++ b/tests/units/resources/implementation.py @@ -90,14 +90,14 @@ class ImplementationTest(tests.Test): self.start_online_client() client = IPCClient() - self.node_volume['context'].create( - guid='context', - type='content', - title='title', - summary='summary', - description='description', - author={'fake': None} - ) + self.node_volume['context'].create({ + 'guid': 'context', + 'type': 'content', + 'title': 'title', + 'summary': 'summary', + 'description': 'description', + 'author': {'fake': None}, + }) impl = {'context': 'context', 'license': 'GPLv3+', @@ -108,7 +108,7 @@ class ImplementationTest(tests.Test): self.assertRaises(http.Forbidden, client.post, ['implementation'], impl) self.assertEqual(0, self.node_volume['implementation'].find()[1]) - self.node_volume['context'].update('context', author={tests.UID: None}) + self.node_volume['context'].update('context', {'author': {tests.UID: None}}) guid = client.post(['implementation'], impl) assert self.node_volume['implementation'].exists(guid) diff --git a/tests/units/resources/volume.py b/tests/units/resources/volume.py index e07858f..687f9c8 100755 --- a/tests/units/resources/volume.py +++ b/tests/units/resources/volume.py @@ -16,24 +16,6 @@ from sugar_network.toolkit import coroutine, util class VolumeTest(tests.Test): - def test_SimulateDeleteEvents(self): - - class Document(Resource): - pass - - events = [] - volume = Volume('db', [Document]) - volume.connect(lambda event: events.append(event)) - - volume['document'].create(guid='guid') - del events[:] - volume['document'].update('guid', layer=['deleted']) - - self.assertEqual([ - {'event': 'delete', 'document': 'document', 'guid': 'guid'}, - ], - events) - def test_Subscribe(self): class Document(Resource): @@ -53,15 +35,13 @@ class VolumeTest(tests.Test): assert event.startswith('data: ') assert event.endswith('\n\n') event = json.loads(event[6:]) - if 'props' in event: - event.pop('props') events.append(event) job = coroutine.spawn(read_events) coroutine.dispatch() - volume['document'].create(guid='guid', prop='value1') + volume['document'].create({'guid': 'guid', 'prop': 'value1'}) coroutine.dispatch() - volume['document'].update('guid', prop='value2') + volume['document'].update('guid', {'prop': 'value2'}) coroutine.dispatch() volume['document'].delete('guid') coroutine.dispatch() @@ -103,15 +83,13 @@ class VolumeTest(tests.Test): assert event.startswith('data: ') assert event.endswith('\n\n') event = json.loads(event[6:]) - if 'props' in event: - event.pop('props') events.append(event) job = coroutine.spawn(read_events) coroutine.dispatch() - volume['document'].create(guid='guid', prop='value1') + volume['document'].create({'guid': 'guid', 'prop': 'value1'}) coroutine.dispatch() - volume['document'].update('guid', prop='value2') + volume['document'].update('guid', {'prop': 'value2'}) coroutine.dispatch() volume['document'].delete('guid') coroutine.dispatch() @@ -158,7 +136,7 @@ class VolumeTest(tests.Test): {'user': {'role': 2, 'order': 0}}, volume['document'].get(guid)['author']) - volume['user'].create(guid='user', color='', pubkey='', name='User') + volume['user'].create({'guid': 'user', 'color': '', 'pubkey': '', 'name': 'User'}) guid = call(cp, method='POST', document='document', content={}, principal='user') self.assertEqual( @@ -176,9 +154,9 @@ class VolumeTest(tests.Test): volume = Volume('db', [User, Document]) cp = TestCommands(volume) - volume['user'].create(guid='user1', color='', pubkey='', name='UserName1') - volume['user'].create(guid='user2', color='', pubkey='', name='User Name2') - volume['user'].create(guid='user3', color='', pubkey='', name='User Name 3') + volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'UserName1'}) + volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User Name2'}) + volume['user'].create({'guid': 'user3', 'color': '', 'pubkey': '', 'name': 'User Name 3'}) guid1 = call(cp, method='POST', document='document', content={}, principal='user1') guid2 = call(cp, method='POST', document='document', content={}, principal='user2') @@ -213,9 +191,9 @@ class VolumeTest(tests.Test): volume = Volume('db', [User, Document]) cp = TestCommands(volume) - volume['user'].create(guid='user1', color='', pubkey='', name='User1') - volume['user'].create(guid='user2', color='', pubkey='', name='User2') - volume['user'].create(guid='user3', color='', pubkey='', name='User3') + volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'}) + volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User2'}) + volume['user'].create({'guid': 'user3', 'color': '', 'pubkey': '', 'name': 'User3'}) guid = call(cp, method='POST', document='document', content={}, principal='user1') call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='user2', role=0) @@ -290,8 +268,8 @@ class VolumeTest(tests.Test): volume = Volume('db', [User, Document]) cp = TestCommands(volume) - volume['user'].create(guid='user1', color='', pubkey='', name='User1') - volume['user'].create(guid='user2', color='', pubkey='', name='User2') + volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'}) + volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User2'}) guid = call(cp, method='POST', document='document', content={}, principal='user1') self.assertEqual([ @@ -353,7 +331,7 @@ class VolumeTest(tests.Test): volume = Volume('db', [User, Document]) cp = TestCommands(volume) - volume['user'].create(guid='user1', color='', pubkey='', name='User1') + volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'}) guid = call(cp, method='POST', document='document', content={}, principal='user1') call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='User2', role=0) @@ -400,8 +378,8 @@ class VolumeTest(tests.Test): volume = Volume('db', [User, Document]) cp = TestCommands(volume) - volume['user'].create(guid='user1', color='', pubkey='', name='User1') - volume['user'].create(guid='user2', color='', pubkey='', name='User2') + volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'}) + volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User2'}) guid = call(cp, method='POST', document='document', content={}, principal='user1') call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='user2') call(cp, method='PUT', document='document', guid=guid, cmd='useradd', user='User3') |