From 3dfa845f17d462fd73b00f93eac6f5ecab4203dc Mon Sep 17 00:00:00 2001 From: Aleksey Lim Date: Sat, 03 May 2014 10:47:41 +0000 Subject: Support 0.8 API --- diff --git a/misc/convert-post b/misc/convert-post new file mode 100755 index 0000000..8246dcd --- /dev/null +++ b/misc/convert-post @@ -0,0 +1,9 @@ +#!/bin/bash + +cp -r /srv/db.save/context/* db/context/ + +for guid in `cat pilot.list`; do + echo -n "{\"seqno\": 1, \"value\": [\"pilot\"]}" > db/context/${guid:0:2}/$guid/layer +done + +rm -rf index diff --git a/misc/convert-pre b/misc/convert-pre new file mode 100755 index 0000000..55f2222 --- /dev/null +++ b/misc/convert-pre @@ -0,0 +1,118 @@ +#!/bin/bash + +rm -f pilot.list +for i in `grep pilot --include=layer db/context/ -Rl`; do echo $(basename `dirname $i`) >> pilot.list; done +echo "{" > var/pilot-releases +for i in `grep pilot --include=layer db/implementation/ -Rl`; do + d=`dirname $i` + c=`cat $d/context | awk -F\" '{print $6}'` + v=`cat $d/version | awk -F\" '{print $6}'` + echo "\"$c\": \"$v\"," >> var/pilot-releases +done +echo '"": null}' >> var/pilot-releases + +rm -rf db/idea db/implementation db/notification db/problem db/question db/sync db/user db/files.index db/guid db/master db/authorization.conf db/report db/user +find db -name index -exec rm -rf {} \; +mkdir -p var +mv db/seqno var/ + +for i in `grep d26cef70447160f31a7497cc0320f23a4e383cc3 --include author -R db | awk -F: '{print $1}'`; do i=`dirname $i`; rm -rf $i; done +for i in `grep package --include type -R db/context | awk -F: '{print $1}'`; do i=`dirname $i`; rm -rf $i; done +find db/context \( -name ef5f33b2378c11e295d00016360ee2af -o -name org.sugarlabs.ContributorHub -o -name edu.mit.media.ScratchActivity -o -name c031aafc143c11e299ed0016360ee2af \) -exec rm -rf {} \; +find db/context \( -name clone -o -name layer -o -name dependencies -o -name downloads -o -name favorite -o -name implement -o -name packages -o -name position -o -name rating -o -name reviews -o -name versions \) -exec rm {} \; +for i in `find db/context -name type`; do sed -i s/project/group/ $i; done +for i in `find db/context -name title`; do p=`dirname $i`; [ -e $p/guid ] || rm -rf $p; done +for i in `find db/artifact -name preview`; do rm -rf `dirname $i`; done +rm -f db/context/7e/7e2c350ac65811e1a30a0016360ee2af/icon* +rm db/context/{layout,seqno} + +for i in `find db/review/ -name guid`; do + src=`dirname $i` + guid=`basename $src` + dst="db/post/${guid:0:2}/$guid" + mkdir -p $dst + mv $src/{author,context,ctime,guid,mtime,seqno,tags,title} $dst/ + mv $src/content $dst/message + echo -n '{"seqno": 1, "value": "review"}' > $dst/type +done +rm -rf db/review + +for i in `find db/artifact/ -name data`; do + src=`dirname $i` + guid=`basename $src` + dst="db/post/${guid:0:2}/$guid" + mkdir -p $dst + mv $src/{author,context,ctime,guid,mtime,seqno,tags,title} $dst/ + mv $src/description $dst/message + echo -n '{"seqno": 1, "value": "file"}' > $dst/type + digest=`grep -o 'digest[^,}]*' $src/data | awk -F\" '{print $3}'` + mime_type=`grep -o 'mime_type[^,}]*' $src/data | awk -F\" '{print $3}'` + blob_size=`ls -l $src/data.blob | awk '{print $5}'` + echo -n "{\"seqno\": 1, \"value\": {\"$digest\": {\"seqno\": 1, \"value\": \"$digest\"}}}" > $dst/attachments + blob_dst=blobs/${digest:0:3}/$digest + mkdir -p `dirname $blob_dst` + mv $src/data.blob $blob_dst + cat >>$blob_dst.meta < $dst/type +done +rm -rf db/feedback + +for i in `find db/solution/ -name guid`; do + src=`dirname $i` + guid=`basename $src` + dst="db/post/${guid:0:2}/$guid" + topic=`grep -o 'value[^,}]*' $src/feedback | awk -F\" '{print $3}'` + mkdir -p $dst + mv $src/{author,context,ctime,guid,mtime,seqno,tags} $dst/ + mv $src/content $dst/message + echo -n "{\"seqno\": 1, \"value\": {}}" > $dst/title + echo -n "{\"seqno\": 1, \"value\": \"solution\"}" > $dst/type + echo -n "{\"seqno\": 1, \"value\": \"$topic\"}" > $dst/topic +done +rm -rf db/solution + +rm -rf comment.list +for i in `find db/comment/ -name guid`; do + src=`dirname $i` + guid=`basename $src` + post=`grep -o 'value[^,}]*' $src/review | awk -F\" '{print $3}'` + if [ -z "$post" -o ! -e db/post/${post:0:2}/$post ]; then + post=`grep -o 'value[^,}]*' $src/solution | awk -F\" '{print $3}'` + if [ -z "$post" -o ! -e db/post/${post:0:2}/$post ]; then + post=`grep -o 'value[^,}]*' $src/feedback | awk -F\" '{print $3}'` + if [ -z "$post" -o ! -e db/post/${post:0:2}/$post ]; then + continue + fi + fi + fi + author=`sed 's/.*value.. //; s/}$//' $src/author` + message=`sed 's/.*value.. //; s/}$//' $src/message` + mtime=`grep -o 'value[^,}]*' $src/mtime | awk '{print $2}'` + mkdir -p comment.list/$post + echo "\"$guid\": {\"author\": $author, \"value\": $message}" > comment.list/$post/$mtime +done +for post in `ls comment.list`; do + value= + for mtime in `ls comment.list/$post`; do + [ "$value" ] && value="$value, " + value="${value}`cat comment.list/$post/$mtime`" + done + echo -n "{\"seqno\": 1, \"value\": {$value}}" > db/post/${post:0:2}/$post/comments +done +rm -rf comment.list +rm -rf db/comment diff --git a/sugar_network/model/__init__.py b/sugar_network/model/__init__.py index d979772..b4ff500 100644 --- a/sugar_network/model/__init__.py +++ b/sugar_network/model/__init__.py @@ -41,6 +41,7 @@ POST_TYPES = [ 'notification', # Auto-generated Post for updates within the Context 'feedback', # Review parent Post 'post', # General purpose dependent Post + 'file', # XXX An attachment, only for compatibility with 0.8 ] STABILITIES = [ diff --git a/sugar_network/model/context.py b/sugar_network/model/context.py index 89a0c08..012e450 100644 --- a/sugar_network/model/context.py +++ b/sugar_network/model/context.py @@ -123,3 +123,19 @@ class Context(db.Resource): continue png = blobs.post(svg_to_png(svg, size), 'image/png').digest self.post(prop, png) + + @db.stored_property(default='') + def implement(self, value): + pass + + @db.stored_property(default='') + def favorite(self, value): + pass + + @db.stored_property(default='') + def clone(self, value): + pass + + @db.indexed_property(db.List, prefix='RL', default=[]) + def layer(self, value): + return value diff --git a/sugar_network/model/post.py b/sugar_network/model/post.py index e26eb91..be417a5 100644 --- a/sugar_network/model/post.py +++ b/sugar_network/model/post.py @@ -33,13 +33,11 @@ class Post(db.Resource): def type(self, value): return value - @db.indexed_property(db.Localized, slot=1, prefix='N', full_text=True, - acl=ACL.CREATE | ACL.READ) + @db.indexed_property(db.Localized, slot=1, prefix='N', full_text=True) def title(self, value): return value - @db.indexed_property(db.Localized, prefix='M', full_text=True, - acl=ACL.CREATE | ACL.READ) + @db.indexed_property(db.Localized, prefix='M', full_text=True) def message(self, value): return value diff --git a/sugar_network/node/master.py b/sugar_network/node/master.py index d5f3f70..78aad76 100644 --- a/sugar_network/node/master.py +++ b/sugar_network/node/master.py @@ -13,17 +13,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import os +import json import logging from urlparse import urlsplit +from os.path import join, exists from sugar_network import toolkit from sugar_network.model.post import Post from sugar_network.model.report import Report from sugar_network.node import model +from sugar_network.node.auth import Principal from sugar_network.node.routes import NodeRoutes -from sugar_network.toolkit.router import route +from sugar_network.toolkit.router import route, ACL from sugar_network.toolkit.coroutine import this -from sugar_network.toolkit import http, packets, pylru, ranges, enforce +from sugar_network.toolkit.spec import format_version +from sugar_network.toolkit import http, packets, pylru, ranges, i18n, enforce RESOURCES = (model.User, model.Context, Post, Report) @@ -36,6 +41,9 @@ class MasterRoutes(NodeRoutes): def __init__(self, master_api, **kwargs): NodeRoutes.__init__(self, urlsplit(master_api).netloc, **kwargs) self._pulls = pylru.lrucache(1024) + self._auth = _Auth() + self._pilot_releases = {} + self._pilot_releases_mtime = -1 @route('POST', cmd='sync', arguments={'accept_length': int}) def sync(self, accept_length): @@ -125,3 +133,493 @@ class MasterRoutes(NodeRoutes): reply.append(('push', None, push)) return reply + + @route('GET', cmd='info', mime_type='application/json') + def info(self): + documents = {} + for name, directory in self.volume.items(): + documents[name] = {'mtime': 0} + return {'guid': self.guid, 'documents': documents} + + @route('POST', ['user'], mime_type='application/json') + def register(self): + request = this.request + for prop in ('color', 'machine_sn', 'machine_uuid'): + if prop in request.content: + del request.content[prop] + user = request.environ.get('HTTP_SUGAR_USER') + this.principal = Principal(user, 0xFF) + request.content['guid'] = user + self.create() + + @route('GET', ['feedback'], + arguments={'offset': int, 'limit': int, 'reply': ('guid',)}, + mime_type='application/json') + def find_feedback(self, reply, limit): + this.request.resource = 'post' + if 'type' not in this.request: + this.request['type'] = ['question', 'problem', 'idea'] + if reply and 'content' in reply: + reply.remove('content') + reply.append('message') + result = NodeRoutes.find(self, reply, limit) + for item in result['result']: + if 'message' in item: + item['content'] = item.pop('message') + if 'author' in item: + item['author'] = _format_author(item) + return result + + @route('GET', ['feedback', None], arguments={'reply': list}, + mime_type='application/json') + def get_feedback(self, reply): + this.request.resource = 'post' + if reply and 'content' in reply: + reply.remove('content') + reply.append('message') + result = NodeRoutes.get(self, reply) + if 'message' in result: + result['content'] = result.pop('message') + if 'author' in result: + result['author'] = _format_author(result) + return result + + @route('GET', ['feedback', None, None], mime_type='application/json') + def get_feedback_prop(self): + this.request.resource = 'post' + if this.request.prop == 'content': + this.request.prop = 'message' + return NodeRoutes.get_prop(self) + + @route('POST', ['feedback'], acl=ACL.AUTH, mime_type='application/json') + def create_feedback(self): + request = this.request + request.resource = 'post' + if 'content' in request.content: + request.content['message'] = request.content.pop('content') + return self.create() + + @route('PUT', ['feedback', None], acl=ACL.AUTH | ACL.AUTHOR) + def update_feedback(self): + request = this.request + request.resource = 'post' + if 'content' in request.content: + request.content['message'] = request.content.pop('content') + return self.update() + + @route('GET', ['review'], + arguments={'offset': int, 'limit': int, 'reply': ('guid',)}, + mime_type='application/json') + def find_review(self, reply, limit): + this.request.resource = 'post' + if reply and 'content' in reply: + reply.remove('content') + reply.append('message') + this.request['type'] = 'review' + result = NodeRoutes.find(self, reply, limit) + for item in result['result']: + if 'message' in item: + item['content'] = item.pop('message') + if 'author' in item: + item['author'] = _format_author(item) + return result + + @route('GET', ['review', None], arguments={'reply': list}, + mime_type='application/json') + def get_review(self, reply): + this.request.resource = 'post' + if reply and 'content' in reply: + reply.remove('content') + reply.append('message') + result = NodeRoutes.get(self, reply) + if 'message' in result: + result['content'] = result.pop('message') + if 'author' in result: + result['author'] = _format_author(result) + return result + + @route('GET', ['review', None, None], mime_type='application/json') + def get_review_prop(self): + request = this.request + request.resource = 'post' + if request.prop == 'content': + request.prop = 'message' + return NodeRoutes.get_prop(self) + + @route('POST', ['review'], acl=ACL.AUTH, mime_type='application/json') + def create_review(self): + request = this.request + request.resource = 'post' + if 'content' in request.content: + request.content['message'] = request.content.pop('content') + if 'rating' in request.content: + request.content['vote'] = request.content.pop('rating') + request.content['type'] = 'review' + return self.create() + + @route('PUT', ['review', None], acl=ACL.AUTH | ACL.AUTHOR) + def update_review(self): + request = this.request + request.resource = 'post' + if 'content' in request.content: + request.content['message'] = request.content.pop('content') + return self.update() + + @route('GET', ['solution'], + arguments={'offset': int, 'limit': int, 'reply': ('guid',)}, + mime_type='application/json') + def find_solution(self, reply, limit): + request = this.request + request.resource = 'post' + if reply: + if 'content' in reply: + reply.remove('content') + reply.append('message') + if 'feedback' in reply: + reply.remove('feedback') + reply.append('topic') + if 'feedback' in request: + request['topic'] = request.pop('feedback') + request['type'] = 'solution' + result = NodeRoutes.find(self, reply, limit) + for item in result['result']: + if 'message' in item: + item['content'] = item.pop('message') + if 'topic' in item: + item['feedback'] = item.pop('topic') + if 'author' in item: + item['author'] = _format_author(item) + return result + + @route('GET', ['solution', None], arguments={'reply': list}, + mime_type='application/json') + def get_solution(self, reply): + this.request.resource = 'post' + if reply: + if 'content' in reply: + reply.remove('content') + reply.append('message') + if 'feedback' in reply: + reply.remove('feedback') + reply.append('topic') + result = NodeRoutes.get(self, reply) + if 'message' in result: + result['content'] = result.pop('message') + if 'topic' in result: + result['feedback'] = result.pop('topic') + if 'author' in result: + result['author'] = _format_author(result) + return result + + @route('GET', ['solution', None, None], mime_type='application/json') + def get_solution_prop(self): + request = this.request + request.resource = 'post' + if request.prop == 'content': + request.prop = 'message' + if request.prop == 'feedback': + request.prop = 'topic' + return NodeRoutes.get_prop(self) + + @route('POST', ['solution'], acl=ACL.AUTH, mime_type='application/json') + def create_solution(self): + request = this.request + request.resource = 'post' + if 'content' in request.content: + request.content['message'] = request.content.pop('content') + if 'feedback' in request.content: + request.content['topic'] = request.content.pop('feedback') + request.content['context'] = \ + this.volume['post'][request.content['topic']]['context'] + request.content['title'] = '' + request.content['type'] = 'solution' + return self.create() + + @route('PUT', ['solution', None], acl=ACL.AUTH | ACL.AUTHOR) + def update_solution(self): + request = this.request + request.resource = 'post' + if 'content' in request.content: + request.content['message'] = request.content.pop('content') + if 'feedback' in request.content: + request.content['topic'] = request.content.pop('feedback') + return self.update() + + @route('GET', ['artifact'], + arguments={'offset': int, 'limit': int, 'reply': ('guid',)}, + mime_type='application/json') + def find_artifact(self, reply, limit): + this.request.resource = 'post' + if reply: + if 'description' in reply: + reply.remove('description') + reply.append('title') + this.request['type'] = 'file' + result = NodeRoutes.find(self, reply, limit) + for item in result['result']: + if 'title' in item: + item['description'] = item['title'] + if 'author' in item: + item['author'] = _format_author(item) + return result + + @route('GET', ['artifact', None], arguments={'reply': list}, + mime_type='application/json') + def get_artifact(self, reply): + this.request.resource = 'post' + if reply: + if 'description' in reply: + reply.remove('description') + reply.append('title') + result = NodeRoutes.get(self, reply) + if 'title' in result: + result['description'] = result['title'] + if 'author' in result: + result['author'] = _format_author(result) + return result + + @route('GET', ['artifact', None, None], mime_type='application/json') + def get_artifact_prop(self): + request = this.request + request.resource = 'post' + if request.prop == 'description': + request.prop = 'title' + elif request.prop == 'data': + attachments = this.volume['post'][request.guid]['attachments'] + enforce(attachments, http.NotFound, 'No attachments') + this.response.content_type = 'application/octet-stream' + blob = this.volume.blobs.get(attachments.values()[0]['value']) + return blob.iter_content() + return NodeRoutes.get_prop(self) + + @route('POST', ['artifact'], acl=ACL.AUTH, mime_type='application/json') + def create_artifact(self): + request = this.request + request.resource = 'post' + if 'description' in request.content: + request.content['title'] = request.content.pop('description') + request.content['message'] = '' + request.content['type'] = 'file' + return self.create() + + @route('PUT', ['artifact', None], acl=ACL.AUTH | ACL.AUTHOR) + def update_artifact(self): + request = this.request + request.resource = 'post' + if 'description' in request.content: + request.content['title'] = request.content.pop('description') + return self.update() + + @route('PUT', ['artifact', None, None], acl=ACL.AUTH | ACL.AUTHOR) + def update_artifact_prop(self): + request = this.request + request.resource = 'post' + if request.prop == 'description': + request.prop = 'title' + elif request.prop == 'data': + request.prop = 'attachments' + request.method = 'POST' + return self.insert_to_aggprop() + return self.update_prop() + + @route('GET', ['comment'], + arguments={'offset': int, 'limit': int, 'reply': ('guid',)}, + mime_type='application/json') + def find_comment(self, reply, limit): + request = this.request + for prop in ('review', 'feedback', 'solution'): + if prop in request: + post = this.volume['post'][request[prop]] + break + else: + return {'total': 0, 'result': []} + result = [] + for key, agg in post['comments'].items(): + if 'value' not in agg: + continue + user, author = agg['author'].items()[0] + author['guid'] = user + result.append({ + 'guid': '%s-%s' % (post.guid, key), + 'message': i18n.decode(agg['value'], request.accept_language), + 'context': post['context'], + 'author': [author], + 'ctime': agg['ctime'], + 'mtime': agg['ctime'], + prop: post.guid, + }) + return {'total': len(result), 'result': result} + + @route('GET', ['comment', None], arguments={'reply': list}, + mime_type='application/json') + def get_comment(self, reply): + request = this.request + guid, key = request.guid.split('-') + post = this.volume['post'][guid] + agg = post['comments'][key] + enforce(agg and 'value' in agg, http.NotFound, 'No such comment') + return {'guid': request.guid, + 'message': i18n.decode(agg['value'], request.accept_language), + 'context': post['context'], + 'author': _format_author(agg), + 'ctime': agg['ctime'], + 'mtime': agg['ctime'], + post['type']: post.guid, + } + + @route('POST', ['comment'], acl=ACL.AUTH, mime_type='application/json') + def create_comment(self): + request = this.request + request.resource = 'post' + for prop in ('review', 'feedback', 'solution'): + if prop in request.content: + request.guid = request.content[prop] + break + else: + raise http.BadRequest('No topic') + request.prop = 'comments' + request.method = 'POST' + request.content = request.content['message'] + key = self.insert_to_aggprop() + return '%s-%s' % (request.guid, key) + + @route('DELETE', ['comment', None], acl=ACL.AUTH | ACL.AUTHOR) + def delete_comment(self): + request = this.request + request.resource = 'post' + request.guid, request.key = request.guid.split('-') + request.prop = 'comments' + self.remove_from_aggprop() + + @route('GET', ['context'], + arguments={'offset': int, 'limit': int, 'reply': ('guid',), 'type': list}, + mime_type='application/json') + def find_context(self, reply, limit): + if reply: + if 'artifact_icon' in reply: + reply.remove('artifact_icon') + reply.append('artefact_icon') + if 'preview' in reply: + reply.remove('preview') + reply.append('logo') + type_ = this.request.setdefault('type', []) + if 'project' in type_: + type_.remove('project') + type_.append('group') + result = NodeRoutes.find(self, reply, limit) + for item in result['result']: + if item.get('type') == ['group']: + item['type'] = ['project'] + if 'artefact_icon' in item: + item['artifact_icon'] = item.pop('artefact_icon') + if 'logo' in item: + item['preview'] = item.pop('logo') + if 'author' in item: + item['author'] = _format_author(item) + return result + + @route('GET', ['context', None], arguments={'reply': list}, + mime_type='application/json') + def get_context(self, reply): + if reply: + if 'artifact_icon' in reply: + reply.remove('artifact_icon') + reply.append('artefact_icon') + if 'preview' in reply: + reply.remove('preview') + reply.append('logo') + result = NodeRoutes.get(self, reply) + if result.get('type') == ['group']: + result['type'] = ['project'] + if 'artefact_icon' in result: + result['artifact_icon'] = result.pop('artefact_icon') + if 'logo' in result: + result['preview'] = result.pop('logo') + if 'author' in result: + result['author'] = _format_author(result) + return result + + @route('GET', ['context', None, None], mime_type='application/json') + def get_context_prop(self): + if this.request.prop == 'artifact_icon': + this.request.prop = 'artefact_icon' + result = NodeRoutes.get_prop(self) + if this.request.prop == 'type' and result == 'group': + return 'project' + return result + + @route('POST', ['context'], acl=ACL.AUTH, mime_type='application/json') + def create_context(self): + request = this.request + if 'project' in request.content['type']: + request.content['type'].remove('project') + request.content['type'].append('group') + if 'artifact_icon' in request.content: + request.content['artefact_icon'] = request.content.pop('artifact_icon') + return self.create() + + @route('GET', ['context', None], cmd='feed', mime_type='application/json') + def feed(self, layer): + context = this.volume['context'][this.request.guid] + implementations = [] + for key, aggvalue in context['releases'].items(): + release = aggvalue['value'] + version = format_version(release['version']) + print layer, context.guid, self._pilot_version(context.guid) + print self._pilot_releases + if layer == 'pilot' and \ + self._pilot_version(context.guid) != version: + continue + implementations.append({ + 'guid': release['bundles']['*-*']['blob'], + 'version': version, + 'arch': '*-*', + 'stability': release['stability'], + 'commands': release['commands'], + 'extract': release['extract'], + }) + return {'name': i18n.decode(context['title'], + this.request.accept_language), + 'implementations': implementations, + } + + @route('GET', ['implementation', None, 'data']) + def get_implementation(self): + return this.volume.blobs.get(this.request.guid) + + def _pilot_version(self, guid): + path = join(this.volume.root, 'var', 'pilot-releases') + if exists(path) and os.stat(path).st_mtime > self._pilot_releases_mtime: + with file(path) as f: + self._pilot_releases = json.load(f) + self._pilot_releases_mtime = os.stat(path).st_mtime + return self._pilot_releases.get(guid) + + +class _Auth(object): + + def __init__(self): + self._authenticated = set() + + def logon(self, request): + user = request.environ.get('HTTP_SUGAR_USER') + enforce(user, http.Unauthorized, 'No credentials') + + if user not in self._authenticated: + _logger.debug('Logging %r user', user) + enforce(this.volume['user'][user].available, http.Unauthorized, + 'Principal user does not exist') + self._authenticated.add(user) + + return Principal(user, 0xFF) + + +def _format_author(value): + result = [] + for guid, props in sorted(value['author'].items(), + key=lambda x: '%016d%s' % (x[1]['role'], x[1].get('name', x[0]))): + if 'name' not in props: + props['name'] = guid + props['guid'] = guid + result.append(props) + return result diff --git a/sugar_network/node/model.py b/sugar_network/node/model.py index 48fa5a3..de31932 100644 --- a/sugar_network/node/model.py +++ b/sugar_network/node/model.py @@ -44,9 +44,7 @@ _presolve_queue = Queue() class User(_user.User): - - def created(self): - self.posts['guid'] = str(hashlib.sha1(self['pubkey']).hexdigest()) + pass class _ReleaseValue(dict): -- cgit v0.9.1