From 743a4627a7742d9da401af78e4078257b2abdb9f Mon Sep 17 00:00:00 2001 From: Aleksey Lim Date: Mon, 01 Oct 2012 05:35:38 +0000 Subject: Form aslo sync script as a regular app and add push functionality --- diff --git a/misc/aslo-sync b/misc/aslo-sync new file mode 100755 index 0000000..70c477d --- /dev/null +++ b/misc/aslo-sync @@ -0,0 +1,458 @@ +#!/usr/bin/env python + +# Copyright (C) 2012 Aleksey Lim +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import time +import getpass +import logging +import tempfile +import subprocess +from base64 import b64encode +from os.path import join, exists + +import MySQLdb as mdb + +import active_document as ad +from sugar_network import Client +from sugar_network.toolkit.collection import PersistentSequence, Sequence +from sugar_network.zerosugar import GOOD_LICENSES, Bundle, parse_version +from sugar_network.resources.volume import Volume +from sugar_network.node import data_root +from sugar_network.local import api_url +from active_toolkit.options import Option +from active_toolkit import application + + +DOWNLOAD_URL = 'http://download.sugarlabs.org/activities' +ACTIVITIES_PATH = '/upload/activities' +EXCLUDE_BUNDLE_IDS = ['net.gcompris'] +SUGAR_GUID = 'sugar' +SN_GUID = 'sugar-network' + +CATEGIORIES_TO_TAGS = { + 'Search & Discovery': 'discovery', + 'Documents': 'documents', + 'Chat, mail and talk': 'conversation', + 'Programming': 'programming', + 'Maps & Geography': 'geography', + 'Media players': 'media', + 'Teacher tools': 'teacher', + 'Games': 'games', + 'Media creation': 'media', + 'Maths & Science': 'science', + 'News': 'news', + } + +MISNAMED_LICENSES = { + ('artistic', '2.0'): 'Artistic 2.0', + ('cc-by-sa',): 'CC-BY-SA', + ('creative', 'share', 'alike'): 'CC-BY-SA', + } + + +class Application(application.Application): + + _my_connection = None + _volume = None + _client = None + + @property + def volume(self): + if self._volume is None: + self._volume = Volume(join(data_root.value, 'db')) + self._volume.populate() + return self._volume + + @property + def client(self): + if self._client is None: + self._client = Client(api_url.value) + return self._client + + def epilog(self): + if self._volume is not None: + self._volume.close() + + @application.command( + 'pull activities.sugarlabs.org content to local db') + def pull(self): + if not self.volume['context'].exists(SN_GUID): + self.volume['context'].create( + guid=SN_GUID, + implement=SN_GUID, + type='project', + title='Sugar Network', + summary='Sugar Network', + description='Sugar Network', + ctime=time.time(), mtime=time.time()) + + if not self.volume['context'].exists(SUGAR_GUID): + self.volume['context'].create( + guid=SUGAR_GUID, + implement=SUGAR_GUID, + type='package', title='Sugar', + summary='Constructionist learning platform', + description= \ + 'Sugar provides simple yet powerful means of engaging ' + 'young children in the world of learning that is ' + 'opened up by computers and the Internet. With Sugar, ' + 'even the youngest learner will quickly become ' + 'proficient in using the computer as a tool to engage ' + 'in authentic problem-solving. Sugar promotes ' + 'sharing, collaborative learning, and reflection, ' + 'developing skills that help them in all aspects ' + 'of life.', + ctime=time.time(), mtime=time.time(), + ) + + if self.args: + for addon_id in self.args: + self.sync_activities(addon_id) + else: + self.sync_activities() + + @application.command( + 'submit pulled activities.sugarlabs.org content to ' + 'Sugar Network server') + def push(self): + pull_seq = PersistentSequence(join(data_root.value, 'push'), [1, None]) + last_seqno = None + try: + for document, directory in self.volume.items(): + for guid, seqno, diff in directory.diff(pull_seq, limit=1024): + for meta in diff.values(): + if 'path' in meta: + with file(meta.pop('path')) as f: + meta['content'] = b64encode(f.read()) + self.client.put([document, guid], diff, cmd='merge') + last_seqno = max(last_seqno, seqno) + finally: + if last_seqno is not None: + pull_seq.exclude(1, last_seqno) + pull_seq.commit() + + def sync_activities(self, addon_id=None): + sql = """ + SELECT + id, + guid + FROM + addons + WHERE + status > 0 AND status < 5 + """ + if addon_id: + sql += ' AND id = %s' % addon_id + + for addon_id, bundle_id in self.sqlexec(sql): + if [i for i in EXCLUDE_BUNDLE_IDS if i in bundle_id]: + continue + self.sync_context(addon_id, bundle_id) + self.sync_versions(addon_id, bundle_id) + + def sync_versions(self, addon_id, bundle_id): + sql = """ + SELECT + versions.version, + addons.status, + licenses.name, + (select max(localized_string) from translations where + id=licenses.text), + versions.created, + versions.releasenotes, + files.filename, + (select version from appversions where + id=applications_versions.min), + (select version from appversions where + id=applications_versions.max) + FROM addons + INNER JOIN versions ON versions.addon_id=addons.id + LEFT JOIN licenses ON licenses.id=versions.license_id + INNER JOIN files ON files.version_id=versions.id + INNER JOIN applications_versions ON + applications_versions.version_id=versions.id + WHERE + addons.status > 0 AND addons.status < 5 AND addons.id = %s + """ % addon_id + + recent_version = None + recent_impl = None + + for version, status, license_id, alicense, release_date, \ + releasenotes, filename, sugar_min, sugar_max \ + in self.sqlexec(sql): + try: + parsed_version = parse_version(version) + except Exception, error: + print '-- Cannot parse %r version for %r: %s' % \ + (version, bundle_id, error) + continue + + if self.volume['implementation'].find( + context=bundle_id, version=version)[1] > 0: + continue + + if license_id is None: + pass + elif license_id == 0: + alicense = 'MPLv1.1' + elif license_id == 1: + alicense = 'GPLv2' + elif license_id == 2: + alicense = 'GPLv3' + elif license_id == 3: + alicense = 'LGPLv2' + elif license_id == 4: + alicense = 'LGPLv3' + elif license_id == 5: + alicense = 'MIT' + elif license_id == 6: + alicense = 'BSD' + else: + parsed_license = self.parse_license(alicense) + if not parsed_license: + print '-- Skip bad license %r from %s' % \ + (alicense, filename) + continue + alicense = parsed_license + + impl = self.sync_implementaiton(bundle_id, + 'http://download.sugarlabs.org/activities/%s/%s' % \ + (addon_id, filename), + sugar_min, sugar_max, + stability='stable' if status == 4 else 'developer', + date=int(time.mktime(release_date.timetuple())), + notes=self.get_i18n_field(releasenotes), + license=[alicense] if alicense else [], + ) + if impl and parsed_version > recent_version: + recent_version = parsed_version + recent_impl = impl + + if recent_version: + icon = recent_impl.pop('artifact_icon') + self.volume['context'].update(bundle_id, **recent_impl) + self.volume['context'].set_blob(bundle_id, 'artifact_icon', icon) + + with tempfile.NamedTemporaryFile() as f: + f.write(icon) + f.flush() + + path = f.name + '.png' + subprocess.check_call([ + 'convert', + '-background', 'none', + '-adaptive-resize', '55x55', + f.name, path]) + with file(path) as icon: + self.volume['context'].set_blob(bundle_id, 'icon', icon) + + path = f.name + '.png' + subprocess.check_call([ + 'convert', + '-background', 'none', + '-density', '400', + '-adaptive-resize', '160x120', + f.name, path]) + with file(path) as icon: + self.volume['context'].set_blob(bundle_id, 'preview', icon) + + def sync_context(self, addon_id, bundle_id): + if not self.volume['context'].exists(bundle_id): + self.volume['context'].create(guid=bundle_id, type='activity', + implement=bundle_id, title='', summary='', description='', + user=['aslo'], layer=['public'], ctime=0, mtime=0) + + created, modified, title, summary, description, homepage, name = \ + self.sqlexec(""" + SELECT + addons.created, + addons.modified, + addons.name, + addons.summary, + addons.description, + (select max(localized_string) from translations where + id=addons.homepage), + CONCAT_WS(' ', users.firstname, users.lastname) + FROM + addons + INNER JOIN addons_users on addons_users.addon_id=addons.id + INNER JOIN users on users.id=addons_users.user_id + WHERE addons.id=%s + """ % addon_id)[0] + created = int(time.mktime(created.timetuple())) + modified = int(time.mktime(modified.timetuple())) + + if self.volume['context'].get(bundle_id)['mtime'] >= modified: + return + print '-- Update %r activity' % bundle_id + + tags = set() + for row in self.sqlexec(""" + SELECT + (select localized_string from translations where + id=categories.name AND locale='en-US') + FROM addons_categories + INNER JOIN categories ON + categories.id=addons_categories.category_id + WHERE + addons_categories.addon_id=%s + """ % addon_id): + tags.add(CATEGIORIES_TO_TAGS[row[0]]) + for row in self.sqlexec(""" + SELECT + tags.tag_text + FROM users_tags_addons + INNER JOIN tags ON tags.id=users_tags_addons.tag_id + INNER JOIN addons_users ON + addons_users.addon_id=users_tags_addons.addon_id + WHERE + users_tags_addons.addon_id=%s + """ % addon_id): + tags.add(row[0]) + + self.volume['context'].update(bundle_id, + title=self.get_i18n_field(title), + summary=self.get_i18n_field(summary), + description=self.get_i18n_field(description), + homepage=homepage or '', + tags=list(tags), + author=[name], + ctime=created, + mtime=modified) + + def sync_implementaiton(self, context, url, sugar_min, sugar_max, + **impl_props): + path = url[len(DOWNLOAD_URL):].strip('/').split('/') + path = join(ACTIVITIES_PATH, *path) + if not exists(path): + print '-- Cannot find ASLO bundle at %s' % path + return None + + try: + bundle = Bundle(path) + except Exception, error: + print '-- Cannor read %s bundle: %s' % (path, error) + return None + + spec = bundle.get_spec() + if spec is None: + print '-- Bundle %s does not contain spec' % path + return None + + if not impl_props['license']: + impl_props['license'] = self.parse_license(spec['license']) + if not impl_props['license']: + print '-- Skip bad license %r from %s' % \ + (spec['license'], path) + return None + + print '-- Add %r version to %r activity' % (spec['version'], context) + + spec.requires[SUGAR_GUID] = { + 'restrictions': [ + (sugar_min, None), + (None, '0.%s' % (int(sugar_max.split('.')[-1]) + 1)), + ], + } + + impl = self.volume['implementation'].create( + context=context, + version=spec['version'], + user=['aslo'], + spec={'*-*': { + 'commands': spec.commands, + 'requires': spec.requires, + 'extract': bundle.extract, + }}, + ctime=time.time(), mtime=time.time(), + **impl_props) + self.volume['implementation'].set_blob(impl, 'data', url=url) + + icon = bundle.extractfile(join(bundle.extract, spec['icon'])).read() + return {'homepage': spec['homepage'] or '', + 'mime_types': spec['mime_types'] or [], + 'artifact_icon': icon, + } + + def parse_license(self, alicense): + for good in GOOD_LICENSES: + if not alicense or good in ['ec']: + continue + if good in alicense: + alicense = good + break + else: + for words, good in MISNAMED_LICENSES.items(): + for i in words: + if i not in alicense.lower(): + break + else: + alicense = good + break + else: + return None + + return alicense + + def get_i18n_field(self, id): + result = {} + if id: + for locale, value in self.sqlexec(""" + SELECT + locale, localized_string + FROM + translations + WHERE + id = %s""" % id): + result[locale] = value + return result + + def sqlexec(self, text): + if self._my_connection is None: + self._my_connection = mdb.connect('localhost', + user.value, getpass.getpass(), database.value) + cursor = self._my_connection.cursor() + cursor.execute(text) + return cursor.fetchall() + + +server = Option( + 'MySQL server', + default='localhost', name='server') +database = Option( + 'MySQL database', + default='activities', name='database') +user = Option( + 'MySQL user', + default='root', name='user') + +Option.seek('main', [application.debug]) +Option.seek('aslo', [server, user, database]) +Option.seek('node', [data_root]) +Option.seek('local', [api_url]) + +ad.index_write_queue.value = 1024 * 10 +ad.index_flush_threshold.value = 0 +ad.index_flush_timeout.value = 0 + +application = Application( + name='sugar-network-aslo', + description= \ + 'Synchronize Sugar Network content with ' + 'http://activities.sugarlabs.org', + config_files=['/etc/sweets.conf', '~/.config/sweets/config']) +application.start() diff --git a/misc/aslo_sync.py b/misc/aslo_sync.py deleted file mode 100755 index b48e0ae..0000000 --- a/misc/aslo_sync.py +++ /dev/null @@ -1,371 +0,0 @@ -#!/usr/bin/env python - -# Copyright (C) 2012 Aleksey Lim -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# sugar-lint: disable - -import os -import sys -import json -import time -import getpass -import logging -from cStringIO import StringIO -from os.path import join, exists - -import MySQLdb as mdb - -import active_document as ad -from sugar_network.zerosugar import GOOD_LICENSES, Bundle -from sugar_network.resources.volume import Volume - - -DOWNLOAD_URL = 'http://download.sugarlabs.org/activities' -ACTIVITIES_PATH = '/upload/activities' -EXCLUDE_BUNDLE_IDS = ['net.gcompris'] -SUGAR_GUID = 'sugar' -SN_GUID = 'sugar' - -CATEGIORIES_TO_TAGS = { - 'Search & Discovery': 'discovery', - 'Documents': 'documents', - 'Chat, mail and talk': 'conversation', - 'Programming': 'programming', - 'Maps & Geography': 'geography', - 'Media players': 'media', - 'Teacher tools': 'teacher', - 'Games': 'games', - 'Media creation': 'media', - 'Maths & Science': 'science', - 'News': 'news', - } - -MISNAMED_LICENSES = { - ('artistic', '2.0'): 'Artistic 2.0', - ('cc-by-sa',): 'CC-BY-SA', - ('creative', 'share', 'alike'): 'CC-BY-SA', - } - -connection = None -volume = None - - -def main(): - if not volume['context'].exists(SN_GUID): - volume['context'].create(guid=SN_GUID, type='project', - title='Sugar Network', summary='Sugar Network', - description='Sugar Network', user=['aslo'], - layer=['public'], ctime=time.time(), mtime=time.time()) - - if not volume['context'].exists(SUGAR_GUID): - volume['context'].create(guid=SUGAR_GUID, type='project', - title='Sugar', summary='Sugar', description='Sugar', - user=['aslo'], - layer=['public'], ctime=time.time(), mtime=time.time()) - - for version in ['0.82', '0.84', '0.86', '0.88', '0.90', '0.92', - '0.94', '0.96']: - guid = '%s-%s' % (SUGAR_GUID, version) - if volume['context'].exists(guid): - continue - volume['implementation'].create(guid=guid, context=SUGAR_GUID, - license=['GPLv3+'], version=version, date=0, - stability='stable', notes='', user=['aslo'], - layer=['public'], ctime=time.time(), mtime=time.time()) - - import_versions() - - -def import_versions(addon_id=None): - rows = sqlexec(""" - SELECT - addons.id, - addons.status, - addons.guid, - licenses.name, - (select max(localized_string) from translations where - id=licenses.text), - versions.created, - (select max(localized_string) from translations where - id=versions.releasenotes), - files.filename, - (select version from appversions where - id=applications_versions.min), - (select version from appversions where - id=applications_versions.max), - CONVERT(versions.version, DECIMAL) - FROM addons - INNER JOIN versions ON versions.addon_id=addons.id - LEFT JOIN licenses ON licenses.id=versions.license_id - INNER JOIN files ON files.version_id=versions.id - INNER JOIN applications_versions ON - applications_versions.version_id=versions.id - WHERE - (select version from appversions where - id=applications_versions.min) < 0.96 AND - addons.status > 0 AND addons.status < 5 - %s - """ % ('AND addons.id = %s' % addon_id if addon_id else '')) - - grouped_rows = {} - for row in rows: - grouped = grouped_rows.get(row[0]) - if grouped is None or grouped[-1] < row[-1]: - grouped_rows[row[0]] = row - - for row in grouped_rows.values(): - addon_id, status, bundle_id, license_id, alicense, release_date, \ - releasenotes, filename, sugar_min, sugar_max, __ = row - if [i for i in EXCLUDE_BUNDLE_IDS if i in bundle_id]: - continue - - if license_id is None: - pass - elif license_id == 0: - alicense = 'MPLv1.1' - elif license_id == 1: - alicense = 'GPLv2' - elif license_id == 2: - alicense = 'GPLv3' - elif license_id == 3: - alicense = 'LGPLv2' - elif license_id == 4: - alicense = 'LGPLv3' - elif license_id == 5: - alicense = 'MIT' - elif license_id == 6: - alicense = 'BSD' - else: - parsed_license = parse_license(alicense) - if not parsed_license: - print '-- Skip bad license %r from %s' % (alicense, filename) - continue - alicense = parsed_license - - if not volume['context'].exists(bundle_id): - context_new(addon_id, bundle_id) - - release_from_aslo(bundle_id, - 'http://download.sugarlabs.org/activities/%s/%s' % \ - (addon_id, filename), - sugar_min, sugar_max, - stability='stable' if status == 4 else 'developer', - date=int(time.mktime(release_date.timetuple())), - notes=releasenotes or 'Mirror activity from ASLO', - license=[alicense] if alicense else [], - ) - - -def context_new(addon_id, bundle_id): - title, summary, description, homepage, nick, name, icondata = sqlexec(""" - SELECT - (select max(localized_string) from translations where - id=addons.name), - (select max(localized_string) from translations where - id=addons.summary), - (select max(localized_string) from translations where - id=addons.description), - (select max(localized_string) from translations where - id=addons.homepage), - users.nickname, - CONCAT_WS(' ', users.firstname, users.lastname), - icondata - FROM - addons - INNER JOIN addons_users on addons_users.addon_id=addons.id - INNER JOIN users on users.id=addons_users.user_id - WHERE addons.id=%s - """ % addon_id)[0] - - tags = set() - for row in sqlexec(""" - SELECT - (select localized_string from translations where - id=categories.name AND locale='en-US') - FROM addons_categories - INNER JOIN categories ON - categories.id=addons_categories.category_id - WHERE - addons_categories.addon_id=%s - """ % addon_id): - tags.add(CATEGIORIES_TO_TAGS[row[0]]) - for row in sqlexec(""" - SELECT - tags.tag_text - FROM users_tags_addons - INNER JOIN tags ON tags.id=users_tags_addons.tag_id - INNER JOIN addons_users ON - addons_users.addon_id=users_tags_addons.addon_id - WHERE - users_tags_addons.addon_id=%s - """ % addon_id): - tags.add(row[0]) - - authors = [] - if nick: - authors.append(nick) - if name: - authors.append(name) - - volume['context'].create(guid=bundle_id, type='activity', - implement=bundle_id, title=title, summary=summary or title, - description=description or title, homepage=homepage or '', - tags=list(tags), user= ['aslo'], author=authors, - layer=['public'], ctime=time.time(), mtime=time.time()) - - if icondata: - volume['context'].set_blob(bundle_id, 'icon', StringIO(icondata)) - - for row in sqlexec(""" - SELECT - thumbdata - FROM previews - WHERE - addon_id=%s - """ % addon_id): - thumb, = row - if thumb: - volume['context'].set_blob(bundle_id, 'preview', StringIO(thumb)) - break - - -def release_from_aslo(context_guid, url, sugar_min, sugar_max, stability, - **kwargs): - path = url[len(DOWNLOAD_URL):].strip('/').split('/') - path = join(ACTIVITIES_PATH, *path) - if not exists(path): - print '-- Cannot find ASLO bundle at %s' % path - return - - try: - bundle = Bundle(path) - except Exception, error: - print '-- Cannor read %s bundle: %s' % (path, error) - return - - spec = bundle.get_spec() - if spec is None: - print '-- Bundle %s does not contain spec' % path - return - - if not kwargs['license']: - kwargs['license'] = parse_license(spec['license']) - if not kwargs['license']: - print '-- Skip bad license %r from %s' % (spec['license'], path) - return - - # TODO - #spec.lint() - - feed_info = volume['context'].get(context_guid).meta('feed') - if not feed_info: - feed = {} - else: - with file(feed_info['path']) as f: - feed = json.load(f) - - kwargs['context'] = context_guid - kwargs['version'] = spec['version'] - kwargs['stability'] = stability - kwargs['user'] = ['aslo'] - impl_guid = volume['implementation'].create(kwargs) - volume['implementation'].set_blob(impl_guid, 'data', url) - - if not feed or spec['version'] >= max(feed.keys()): - volume['context'].update(context_guid, { - 'type': spec.types, - 'title': spec['name'], - 'summary': spec['summary'], - 'description': spec['description'], - 'homepage': spec['homepage'] or '', - 'tags': spec['tags'] or [], - 'mime_types': spec['mime_types'] or [], - }) - - icon_path = join(bundle.extract, spec['icon']) - try: - volume['context'].set_blob(context_guid, 'artifact_icon', - bundle.extractfile(icon_path)) - except Exception, error: - print '-- No icon for %s: %s' % (path, icon_path) - - sugar_dep = { - 'restrictions': [ - (sugar_min, None), - (None, '0.%s' % (int(sugar_max.split('.')[-1]) + 1)), - ], - } - #spec.requires[SUGAR_GUID] = sugar_dep - - feed.setdefault(spec['version'], {}) - feed[spec['version']]['*-*'] = { - 'guid': impl_guid, - 'stability': stability, - 'commands': spec.commands, - 'requires': spec.requires, - 'extract': bundle.extract, - 'size': os.stat(path).st_size, - } - - volume['context'].set_blob(context_guid, 'feed', - StringIO(json.dumps(feed, indent=4))) - - -def parse_license(alicense): - for good in GOOD_LICENSES: - if not alicense or good in ['ec']: - continue - if good in alicense: - alicense = good - break - else: - for words, good in MISNAMED_LICENSES.items(): - for i in words: - if i not in alicense.lower(): - break - else: - alicense = good - break - else: - return None - - return alicense - - -def sqlexec(text): - cur = connection.cursor() - cur.execute(text) - return cur.fetchall() - - -logging.basicConfig(level=logging.INFO) - -ad.index_write_queue.value = 1024 * 10 -ad.index_flush_threshold.value = 0 -ad.index_flush_timeout.value = 0 - -connection = mdb.connect('localhost', - 'root', getpass.getpass(), 'activities') -volume = Volume('db') - -try: - if len(sys.argv) > 1: - for addon_id in sys.argv[1:]: - import_versions(addon_id) - else: - main() -finally: - volume.close() diff --git a/sugar_network/node/commands.py b/sugar_network/node/commands.py index 56a3b93..f0226c7 100644 --- a/sugar_network/node/commands.py +++ b/sugar_network/node/commands.py @@ -163,6 +163,12 @@ class MasterCommands(NodeCommands, SyncCommands): NodeCommands.__init__(self, volume) SyncCommands.__init__(self) + @ad.document_command(method='PUT', cmd='merge', + permissions=ad.ACCESS_AUTH) + def merge(self, document, guid, request): + directory = self.volume[document] + directory.merge(guid, request.content) + def _load_pubkey(pubkey): pubkey = pubkey.strip() diff --git a/sugar_network/resources/implementation.py b/sugar_network/resources/implementation.py index b5dc774..b759d6b 100644 --- a/sugar_network/resources/implementation.py +++ b/sugar_network/resources/implementation.py @@ -68,6 +68,10 @@ class Implementation(Resource): def notes(self, value): return value + @ad.active_property(ad.StoredProperty, typecast=dict, default={}) + def spec(self, value): + return value + @ad.active_property(ad.BlobProperty) def data(self, stat): return stat -- cgit v0.9.1