Actualize ASLO sync script

8 files changed, 233 insertions, 166 deletions

diff --git a/misc/aslo-patch-versions b/misc/aslo-patch-versions
index 37d2710..ec6b464 100755
--- a/misc/aslo-patch-versions
+++ b/misc/aslo-patch-versions
@@ -20,9 +20,8 @@ import os
 from optparse import OptionParser
 from os.path import exists
 
-from sugar_network import db
+from sugar_network import db, model
 from sugar_network.node import data_root
-from sugar_network.resources.volume import Volume
 from sugar_network.toolkit.bundle import Bundle
 from sugar_network.toolkit import Option
 
@@ -37,7 +36,7 @@ db.index_write_queue.value = 1024 * 10
 db.index_flush_threshold.value = 0
 db.index_flush_timeout.value = 0
 
-volume = Volume(data_root.value)
+volume = db.Volume(data_root.value, model.RESOURCES)
 volume.populate()
 directory = volume['implementation']
 try:
@@ -50,7 +49,9 @@ try:
         blob_path = '/upload/activities' + url[len(DOWNLOAD_URL):]
         if not exists(blob_path):
             if 'deleted' not in impl['layer']:
-                print '-- Cannot find %r' % blob_path
+                print '-- Delete missed %r' % blob_path
+                impl['layer'].append('deleted')
+                directory.update(impl.guid, {'layer': impl['layer']})
             continue
         unpack_size = 0
         with Bundle(blob_path, mime_type='application/zip') as bundle:
diff --git a/misc/aslo-sync b/misc/aslo-sync
index 3a12c77..00d70da 100755
--- a/misc/aslo-sync
+++ b/misc/aslo-sync
@@ -16,26 +16,29 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import sys
 import time
 import shutil
 import getpass
 import gettext
 import tempfile
+import traceback
 from os.path import join
 
 import MySQLdb as mdb
 
-from sugar_network import db, client
+from sugar_network import db, client, model, toolkit
 from sugar_network.node import data_root
 from sugar_network.client.bundle import Bundle
-from sugar_network.resources.volume import Volume
-from sugar_network.node.slave import SlaveCommands
+from sugar_network.node.slave import SlaveRoutes
 from sugar_network.node.routes import load_bundle
 from sugar_network.toolkit import util, licenses, application, spec, Option
 
 
 DOWNLOAD_URL = 'http://download.sugarlabs.org/activities'
-ASLO_GUID = 'd26cef70447160f31a7497cc0320f23a4e383cc3'
+ASLO_AUTHOR = {'d26cef70447160f31a7497cc0320f23a4e383cc3': {
+    'order': 0, 'role': 1, 'name': 'Activity Library',
+    }}
 ACTIVITIES_PATH = '/upload/activities'
 SUGAR_GUID = 'sugar'
 SN_GUID = 'sugar-network'
@@ -53,6 +56,9 @@ CATEGIORIES_TO_TAGS = {
         'Media creation': 'media',
         'Maths & Science': 'science',
         'News': 'news',
+        'Utilities': 'utilities',
+        'Web': 'web',
+        'Communications and Language': 'literacy',
         }
 
 MISNAMED_LICENSES = {
@@ -76,6 +82,7 @@ IGNORE_VERSIONS = frozenset([
         29464,  # No file
         30074,  # No file
         30234,  # No file
+        31809,  # rsvg fails to load icon
         ])
 
 LICENSES_MAP = {
@@ -125,7 +132,7 @@ class Application(application.Application):
     @property
     def volume(self):
         if self._volume is None:
-            self._volume = Volume(data_root.value)
+            self._volume = db.Volume(data_root.value, model.RESOURCES)
             self._volume.populate()
         return self._volume
 
@@ -158,7 +165,7 @@ class Application(application.Application):
                     'description': 'Sugar Network',
                     'ctime': time.time(),
                     'mtime': time.time(),
-                    'author': self.authors(),
+                    'author': ASLO_AUTHOR,
                     })
 
         if not self.volume['context'].exists(SUGAR_GUID):
@@ -180,7 +187,7 @@ class Application(application.Application):
                         'of life.',
                     'ctime': time.time(),
                     'mtime': time.time(),
-                    'author': self.authors(),
+                    'author': ASLO_AUTHOR,
                     })
 
         if not self.volume['context'].exists(PACKAGES_GUID):
@@ -193,10 +200,11 @@ class Application(application.Application):
                     'description': 'Collection of GNU/Linux packages metadata',
                     'ctime': time.time(),
                     'mtime': time.time(),
-                    'author': self.authors(),
+                    'author': ASLO_AUTHOR,
+                    'icon': {
+                        'url': '/static/images/package.png',
+                        },
                     })
-            self.volume['context'].set_blob(PACKAGES_GUID, 'icon',
-                    url='/static/images/package.png')
 
         if self.args:
             for addon_id in self.args:
@@ -208,12 +216,13 @@ class Application(application.Application):
             'submit pulled activities.sugarlabs.org content to '
             'Sugar Network server')
     def push(self):
-        node = SlaveCommands(join(data_root.value, 'node.key'), self.volume)
+        node = SlaveRoutes(join(data_root.value, 'key'), self.volume)
         node.online_sync(no_pull=True)
 
     def sync_activities(self, addon_id=None):
         directory = self.volume['context']
-        items, __ = directory.find(type='activity', layer='public')
+        items, __ = directory.find(type='activity', guid=addon_id,
+                not_layer='deleted')
         existing_activities = set([i.guid for i in items])
 
         sql = """
@@ -235,18 +244,19 @@ class Application(application.Application):
                 self.sync_context(addon_id, bundle_id)
                 self.sync_versions(addon_id, bundle_id)
                 self.sync_reviews(addon_id, bundle_id)
-            except Exception, error:
-                print '-- Cannot sync %r addon: %s' % (addon_id, error)
+            except Exception:
+                print '-- Failed to sync %s addon' % addon_id
+                traceback.print_exception(*sys.exc_info())
             if bundle_id in existing_activities:
                 existing_activities.remove(bundle_id)
 
         for guid in existing_activities:
-            print '-- Hide %r addon deleted on ASLO' % guid
+            print '-- Hide %r deleted activity' % guid
             directory.update(guid, {'layer': ['deleted']})
 
     def sync_reviews(self, addon_id, bundle_id):
         directory = self.volume['review']
-        items, __ = directory.find(context=bundle_id, layer='public')
+        items, __ = directory.find(context=bundle_id, not_layer='deleted')
         existing_reviews = set([i.guid for i in items])
 
         sql = """
@@ -256,10 +266,9 @@ class Application(application.Application):
                 reviews.title,
                 reviews.body,
                 reviews.rating,
+                users.email,
                 users.nickname,
-                IF(users.firstname!="",
-                    CONCAT_WS(' ', users.firstname, users.lastname),
-                    users.nickname)
+                CONCAT_WS(' ', users.firstname, users.lastname)
             FROM
                 reviews
                 INNER JOIN versions ON versions.id = reviews.version_id
@@ -267,12 +276,17 @@ class Application(application.Application):
             WHERE
                 reply_to IS NULL AND versions.addon_id = %s
             """ % addon_id
-        for guid, created, title, content, rating, nickname, author in \
-                self.sqlexec(sql):
+        for guid, created, title, content, rating, email, nickname, fullname \
+                in self.sqlexec(sql):
             guid = str(guid)
             if directory.exists(guid):
                 existing_reviews.remove(guid)
                 continue
+            if not nickname:
+                nickname = email.split('@')[0]
+            fullname = fullname.strip()
+            if not fullname:
+                fullname = nickname
             directory.create({
                     'guid': guid,
                     'ctime': int(time.mktime(created.timetuple())),
@@ -281,16 +295,18 @@ class Application(application.Application):
                     'title': self.get_i18n_field(title),
                     'content': self.get_i18n_field(content),
                     'rating': rating,
-                    'author': self.authors(nickname, author),
+                    'author': {nickname: {
+                        'order': 0, 'role': 3, 'name': fullname,
+                        }},
                     })
 
         for guid in existing_reviews:
-            print '-- Hide %r review deleted on ASLO' % guid
+            print '-- Hide %s %s deleted review' % (bundle_id, guid)
             directory.update(guid, {'layer': ['deleted']})
 
     def sync_versions(self, addon_id, bundle_id):
         directory = self.volume['implementation']
-        items, __ = directory.find(context=bundle_id, layer='public')
+        items, __ = directory.find(context=bundle_id, not_layer='deleted')
         existing_versions = set([i.guid for i in items])
 
         sql = """
@@ -306,34 +322,40 @@ class Application(application.Application):
                 (select version from appversions where
                     id=applications_versions.min),
                 (select version from appversions where
-                    id=applications_versions.max)
+                    id=applications_versions.max),
+                users.email,
+                users.nickname,
+                CONCAT_WS(' ', users.firstname, users.lastname),
+                addons.status
             FROM addons
                 INNER JOIN versions ON versions.addon_id=addons.id
                 LEFT JOIN licenses ON licenses.id=versions.license_id
                 INNER JOIN files ON files.version_id=versions.id
                 INNER JOIN applications_versions ON
                     applications_versions.version_id=versions.id
+                INNER JOIN users ON users.id=versions.uploader
             WHERE
                 addons.status > 0 AND addons.status < 5 AND addons.id = %s
             ORDER BY
-                versions.id DESC
+                versions.id ASC
             """ % addon_id
 
         for version_id, version, license_id, alicense, release_date, \
-                releasenotes, filename, sugar_min, sugar_max \
-                in self.sqlexec(sql):
+                releasenotes, filename, sugar_min, sugar_max, \
+                email, nickname, fullname, status in self.sqlexec(sql):
             if version_id in IGNORE_VERSIONS:
                 continue
+            version_id = str(version_id)
 
             if filename.endswith('.xol'):
-                print '-- Ignore library bundles for %r' % bundle_id
+                print '-- Ignore %r library bundle' % filename
                 continue
 
             try:
                 parsed_version = util.parse_version(version)
             except Exception, error:
                 print '-- Cannot parse %r version for %r: %s' % \
-                        (version, bundle_id, error)
+                        (version, filename, error)
                 continue
 
             if license_id is None:
@@ -359,49 +381,65 @@ class Application(application.Application):
                 elif bundle_id in LICENSES_MAP:
                     alicense = LICENSES_MAP[bundle_id]
                 else:
-                    print '-- Skip bad %r license from %s in %s' % \
-                            (alicense, filename, addon_id)
+                    print '-- Skip %r bad %r license' % (filename, alicense)
                     continue
-
-            items, __ = directory.find(context=bundle_id,
-                    version=version, limit=1)
-            if items:
-                for i in items:
-                    existing_versions.remove(i.guid)
+            if not alicense and bundle_id in LICENSES_MAP:
+                alicense = LICENSES_MAP[bundle_id]
+
+            layers = ['origin']
+            if status == 4:
+                layers.append('public')
+
+            if not nickname:
+                nickname = email.split('@')[0]
+            fullname = fullname.strip()
+            if not fullname:
+                fullname = nickname
+
+            if directory.exists(version_id):
+                if set(directory.get(version_id).layer) != set(layers) or \
+                        version_id not in existing_versions:
+                    directory.update(version_id, {'layer': layers})
+                if version_id in existing_versions:
+                    existing_versions.remove(version_id)
                 continue
 
+            bundle_path = join(ACTIVITIES_PATH, str(addon_id), filename)
             try:
-                self.sync_implementaiton(bundle_id, addon_id, filename,
-                        sugar_min, sugar_max, status, alicense,
-                        self.get_i18n_field(releasenotes),
-                        int(time.mktime(release_date.timetuple())),
-                        )
+                with load_bundle(
+                        self.volume, Request(self.volume, {
+                            'requires': 'sugar>=%s<=%s' % (sugar_min, sugar_max),
+                            'license': alicense,
+                            }),
+                        bundle_path) as impl:
+                    impl['guid'] = version_id
+                    if 'notes' not in impl:
+                        impl['notes'] = self.get_i18n_field(releasenotes)
+                    impl['stability'] = 'stable'
+                    impl['ctime'] = int(time.mktime(release_date.timetuple()))
+                    impl['mtime'] = time.time()
+                    impl['author'] = {nickname: {
+                        'order': 0, 'role': 3, 'name': fullname,
+                        }}
+                    impl['layer'] = layers
+                    impl['data']['url'] = \
+                            '/'.join([DOWNLOAD_URL, str(addon_id), filename])
+                    impl['data']['blob_size'] = os.stat(bundle_path).st_size
             except Exception, error:
-                print '-- Failed to sync %s for %s: %s' % \
-                        (version, bundle_id, error)
-                continue
+                print '-- Failed to sync %r' % filename
+                traceback.print_exception(*sys.exc_info())
+            else:
+                print '-- Sync %r' % filename
 
         for guid in existing_versions:
-            print '-- Hide %s %r version deleted on ASLO' % (bundle_id, guid)
+            print '-- Hide %s %s deleted version' % (bundle_id, guid)
             directory.update(guid, {'layer': ['deleted']})
 
     def sync_context(self, addon_id, bundle_id):
-        if not self.volume['context'].exists(bundle_id):
-            self.volume['context'].create({
-                    'guid': bundle_id,
-                    'type': 'activity',
-                    'implement': bundle_id,
-                    'title': {},
-                    'summary': {},
-                    'description': {},
-                    'author': self.authors(),
-                    'layer': ['public'],
-                    'ctime': 0,
-                    'mtime': 0,
-                    })
+        directory = self.volume['context']
 
-        created, modified, title, summary, description, homepage, nickname, \
-                author = self.sqlexec("""
+        created, modified, title, summary, description, homepage = \
+                self.sqlexec("""
             SELECT
                 addons.created,
                 addons.modified,
@@ -409,23 +447,18 @@ class Application(application.Application):
                 addons.summary,
                 addons.description,
                 (select max(localized_string) from translations where
-                    id=addons.homepage),
-                users.nickname,
-                IF(users.firstname != '',
-                    CONCAT_WS(' ', users.firstname, users.lastname),
-                    users.nickname)
+                    id=addons.homepage)
             FROM
                 addons
-                INNER JOIN addons_users on addons_users.addon_id=addons.id
-                INNER JOIN users on users.id=addons_users.user_id
-            WHERE addons.id=%s
+            WHERE
+                addons.id=%s
             """ % addon_id)[0]
         created = int(time.mktime(created.timetuple()))
         modified = int(time.mktime(modified.timetuple()))
 
-        if self.volume['context'].get(bundle_id)['mtime'] >= modified:
+        if directory.exists(bundle_id) and \
+                directory.get(bundle_id)['mtime'] >= modified:
             return
-        print '-- Update %r activity' % bundle_id
 
         tags = set()
         for row in self.sqlexec("""
@@ -451,40 +484,46 @@ class Application(application.Application):
                 """ % addon_id):
             tags.add(row[0])
 
-        self.volume['context'].update(bundle_id, {
+        authors = {}
+        for order, (role, email, nickname, fullname) in enumerate(self.sqlexec(
+                """
+                SELECT
+                    addons_users.role,
+                    users.email,
+                    users.nickname,
+                    CONCAT_WS(' ', users.firstname, users.lastname)
+                FROM
+                    addons_users
+                    INNER JOIN users on users.id=addons_users.user_id
+                WHERE
+                    addons_users.addon_id=%s
+                ORDER BY
+                    position
+                """ % addon_id)):
+            if not nickname:
+                nickname = email.split('@')[0]
+            fullname = fullname.strip()
+            if not fullname:
+                fullname = nickname
+            authors[nickname] = {
+                    'order': order,
+                    'role': 3 if role == 5 else 1,
+                    'name': fullname,
+                    }
+
+        directory.update(bundle_id, {
+                'guid': bundle_id,
+                'type': 'activity',
                 'title': self.get_i18n_field(title),
                 'summary': self.get_i18n_field(summary),
                 'description': self.get_i18n_field(description),
                 'homepage': homepage or '',
                 'tags': list(tags),
-                'author': self.authors(nickname, author),
+                'author': authors,
                 'ctime': created,
                 'mtime': modified,
                 })
-
-    def sync_implementaiton(self, context, addon_id, filename,
-            sugar_min, sugar_max, status, license, notes, date):
-        bundle_path = join(ACTIVITIES_PATH, str(addon_id), filename)
-        with load_bundle(self.volume, ASLO_GUID, bundle_path,
-                requires='sugar>=%s; sugar<=%s' % (sugar_min, sugar_max),
-                ) as impl:
-            if impl['license'] == spec.EMPTY_LICENSE:
-                if not license and context in LICENSES_MAP:
-                    license = LICENSES_MAP[context]
-                if not license:
-                    raise Exception('No licenses for %r' % filename)
-                impl['license'] = license
-            if 'notes' not in impl:
-                impl['notes'] = notes
-            impl['stability'] = 'stable' if status == 4 else 'developer'
-            impl['ctime'] = date
-            impl['mtime'] = time.time()
-            impl['author'] = self.authors()
-            impl['data']['url'] = \
-                    '/'.join([DOWNLOAD_URL, str(addon_id), filename])
-            impl['data']['blob_size'] = os.stat(bundle_path).st_size
-
-        print '-- Add %r version to %r activity' % (impl['version'], context)
+        print '-- Sync %r activity' % bundle_id
 
     def parse_license(self, alicense):
         for good in licenses.GOOD_LICENSES:
@@ -532,17 +571,20 @@ class Application(application.Application):
         cursor.execute(text)
         return cursor.fetchall()
 
-    def authors(self, nickname=None, original=None):
-        result = {
-                ASLO_GUID: {
-                    'role': 1, 'order': 0, 'name': 'Activity Library',
-                    },
-                }
-        if nickname:
-            result[nickname] = {
-                    'role': 2, 'order': 1, 'name': original,
-                    }
-        return result
+
+class Request(dict):
+
+    def __init__(self, volume, props):
+        dict.__init__(self, props)
+        self._volume = volume
+
+    def call(self, method, path, content):
+        if method == 'POST':
+            resource, = path
+            return self._volume[resource].create(content)
+        elif method == 'PUT':
+            resource, guid = path
+            self._volume[resource].update(guid, content)
 
 
 mysql_server = Option(
@@ -558,7 +600,7 @@ mysql_password = Option(
         'MySQL password',
         name='mysql_password')
 
-Option.seek('main', [application.debug])
+Option.seek('main', [application.debug, toolkit.cachedir])
 Option.seek('aslo', [mysql_server, mysql_user, mysql_password, mysql_database])
 Option.seek('node', [data_root])
 Option.seek('client', [client.api_url])
diff --git a/sugar_network/db/directory.py b/sugar_network/db/directory.py
index 6bb2d70..60d5c57 100644
--- a/sugar_network/db/directory.py
+++ b/sugar_network/db/directory.py
@@ -236,11 +236,14 @@ class Directory(object):
             if isinstance(self.metadata[prop], StoredProperty) and \
                     self.metadata[prop].localized:
                 if isinstance(value, dict):
-                    orig_value = dict([(i, orig[prop].get(i)) for i in value])
-                    if orig_value == value:
+                    if value == dict([(i, orig[prop].get(i)) for i in value]):
                         continue
                 elif orig.get(prop, accept_language) == value:
                     continue
+            elif isinstance(self.metadata[prop], BlobProperty) and \
+                    isinstance(value, dict) and \
+                    value.get('digest') == orig[prop].get('digest'):
+                continue
             patch[prop] = value
         return patch
 
diff --git a/sugar_network/model/context.py b/sugar_network/model/context.py
index 053228f..e885b53 100644
--- a/sugar_network/model/context.py
+++ b/sugar_network/model/context.py
@@ -13,7 +13,6 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from cStringIO import StringIO
 from os.path import join
 
 from sugar_network import db, model, static
@@ -127,41 +126,3 @@ class Context(db.Resource):
     @db.stored_property(typecast=dict, default={}, acl=ACL.PUBLIC | ACL.LOCAL)
     def packages(self, value):
         return value
-
-    @staticmethod
-    def image_props(svg):
-        icon = StringIO(svg.read())
-        return {'artifact_icon': {
-                    'blob': icon,
-                    'mime_type': 'image/svg+xml',
-                    },
-                'icon': {
-                    'blob': _svg_to_png(icon.getvalue(), 55, 55),
-                    'mime_type': 'image/png',
-                    },
-                'preview': {
-                    'blob': _svg_to_png(icon.getvalue(), 160, 120),
-                    'mime_type': 'image/png',
-                    },
-                }
-
-
-def _svg_to_png(data, w, h):
-    import rsvg
-    import cairo
-
-    svg = rsvg.Handle(data=data)
-    surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, w, h)
-    context = cairo.Context(surface)
-
-    scale = min(float(w) / svg.props.width, float(h) / svg.props.height)
-    context.translate(
-            int(w - svg.props.width * scale) / 2,
-            int(h - svg.props.height * scale) / 2)
-    context.scale(scale, scale)
-    svg.render_cairo(context)
-
-    result = StringIO()
-    surface.write_to_png(result)
-    result.seek(0)
-    return result
diff --git a/sugar_network/node/routes.py b/sugar_network/node/routes.py
index 80caba4..9b0e27f 100644
--- a/sugar_network/node/routes.py
+++ b/sugar_network/node/routes.py
@@ -19,13 +19,13 @@ import shutil
 import gettext
 import logging
 import hashlib
+from cStringIO import StringIO
 from contextlib import contextmanager
 from ConfigParser import ConfigParser
 from os.path import join, isdir, exists
 
 from sugar_network import node, toolkit, model
 from sugar_network.node import stats_node, stats_user
-from sugar_network.model.context import Context
 # pylint: disable-msg=W0611
 from sugar_network.toolkit.router import route, preroute, postroute, ACL
 from sugar_network.toolkit.router import Unauthorized, Request, fallbackroute
@@ -418,7 +418,6 @@ def load_bundle(volume, request, bundle_path):
     if 'initial' in impl:
         initial = impl.pop('initial')
     data = impl.setdefault('data', {})
-    data['blob'] = bundle_path
     contexts = volume['context']
     context = impl.get('context')
     context_meta = None
@@ -445,7 +444,8 @@ def load_bundle(volume, request, bundle_path):
         context = impl['context'] = spec['context']
         impl['version'] = spec['version']
         impl['stability'] = spec['stability']
-        impl['license'] = spec['license']
+        if spec['license'] is not EMPTY_LICENSE:
+            impl['license'] = spec['license']
         data['spec'] = {'*-*': {
             'commands': spec.commands,
             'requires': spec.requires,
@@ -463,25 +463,37 @@ def load_bundle(volume, request, bundle_path):
     enforce('version' in impl, 'Version is not specified')
     enforce(context_type in contexts.get(context)['type'],
             http.BadRequest, 'Inappropriate bundle type')
-    if impl.get('license') in (None, EMPTY_LICENSE):
+    if 'license' not in impl:
         existing, total = impls.find(
                 context=context, order_by='-version', not_layer='deleted')
         enforce(total, 'License is not specified')
         impl['license'] = next(existing)['license']
 
+    digest = hashlib.sha1()
+    with file(bundle_path, 'rb') as f:
+        while True:
+            chunk = f.read(toolkit.BUFFER_SIZE)
+            if not chunk:
+                break
+            digest.update(chunk)
+    data['digest'] = digest.hexdigest()
+
     yield impl
 
     existing, __ = impls.find(
             context=context, version=impl['version'], not_layer='deleted')
+    if 'url' not in data:
+        data['blob'] = bundle_path
     impl['guid'] = \
             request.call(method='POST', path=['implementation'], content=impl)
     for i in existing:
         layer = i['layer'] + ['deleted']
         impls.update(i.guid, {'layer': layer})
 
-    patch = contexts.patch(context, context_meta)
-    if patch and 'origin' in impls.get(impl['guid']).layer:
-        request.call(method='PUT', path=['context', context], content=patch)
+    if 'origin' in impls.get(impl['guid']).layer:
+        diff = contexts.patch(context, context_meta)
+        if diff:
+            request.call(method='PUT', path=['context', context], content=diff)
 
 
 def _load_context_metadata(bundle, spec):
@@ -491,8 +503,18 @@ def _load_context_metadata(bundle, spec):
             result[prop] = spec[prop]
 
     try:
-        with bundle.extractfile(join(bundle.rootdir, spec['icon'])) as svg:
-            result.update(Context.image_props(svg))
+        icon_file = bundle.extractfile(join(bundle.rootdir, spec['icon']))
+        icon = StringIO(icon_file.read())
+        icon_file.close()
+        result.update({
+            'artifact_icon': {
+                'blob': icon,
+                'mime_type': 'image/svg+xml',
+                'digest': hashlib.sha1(icon.getvalue()).hexdigest(),
+                },
+            'preview': _svg_to_png(icon.getvalue(), 160, 120),
+            'icon': _svg_to_png(icon.getvalue(), 55, 55),
+            })
     except Exception:
         exception(_logger, 'Failed to load icon')
 
@@ -518,10 +540,35 @@ def _load_context_metadata(bundle, spec):
                 i18n = gettext.translation(spec['context'],
                         join(tmpdir, *mo_path[:2]), [lang])
                 for prop, value in msgids.items():
-                    msgstr = i18n.gettext(value)
-                    if msgstr != value or lang == 'en':
+                    msgstr = i18n.gettext(value).decode('utf8')
+                    if lang == 'en' or msgstr != value:
                         result[prop][lang] = msgstr
             except Exception:
                 exception(_logger, 'Gettext failed to read %r', mo_path[-1])
 
     return result
+
+
+def _svg_to_png(data, w, h):
+    import rsvg
+    import cairo
+
+    svg = rsvg.Handle(data=data)
+    surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, w, h)
+    context = cairo.Context(surface)
+
+    scale = min(float(w) / svg.props.width, float(h) / svg.props.height)
+    context.translate(
+            int(w - svg.props.width * scale) / 2,
+            int(h - svg.props.height * scale) / 2)
+    context.scale(scale, scale)
+    svg.render_cairo(context)
+
+    result = StringIO()
+    surface.write_to_png(result)
+    result.seek(0)
+
+    return {'blob': result,
+            'mime_type': 'image/png',
+            'digest': hashlib.sha1(result.getvalue()).hexdigest(),
+            }
diff --git a/tests/units/client/implementations.py b/tests/units/client/implementations.py
index d8173e9..06902a0 100755
--- a/tests/units/client/implementations.py
+++ b/tests/units/client/implementations.py
@@ -9,6 +9,7 @@ import pickle
 import shutil
 import zipfile
 import logging
+import hashlib
 from cStringIO import StringIO
 from os.path import exists, dirname
 
@@ -162,6 +163,7 @@ class Implementations(tests.Test):
             'data': {
                 'unpack_size': len(activity_info),
                 'blob_size': len(blob),
+                'digest': hashlib.sha1(blob).hexdigest(),
                 'mime_type': 'application/vnd.olpc-sugar',
                 'spec': {'*-*': {'commands': {'activity': {'exec': 'true'}}, 'requires': {}}},
                 },
diff --git a/tests/units/client/offline_routes.py b/tests/units/client/offline_routes.py
index ac7af1b..2300b7a 100755
--- a/tests/units/client/offline_routes.py
+++ b/tests/units/client/offline_routes.py
@@ -2,6 +2,7 @@
 # sugar-lint: disable
 
 import json
+import hashlib
 from cStringIO import StringIO
 from zipfile import ZipFile
 from os.path import exists
@@ -311,6 +312,7 @@ class OfflineRoutes(tests.Test):
             'data': {
                 'unpack_size': len(activity_info),
                 'blob_size': len(blob),
+                'digest': hashlib.sha1(blob).hexdigest(),
                 'mime_type': 'application/vnd.olpc-sugar',
                 'spec': {'*-*': {'commands': {'activity': {'exec': 'true'}}, 'requires': {}}},
                 },
diff --git a/tests/units/client/online_routes.py b/tests/units/client/online_routes.py
index ee2efe8..6314948 100755
--- a/tests/units/client/online_routes.py
+++ b/tests/units/client/online_routes.py
@@ -7,6 +7,7 @@ import time
 import copy
 import shutil
 import zipfile
+import hashlib
 from zipfile import ZipFile
 from cStringIO import StringIO
 from os.path import exists, lexists, basename
@@ -666,6 +667,7 @@ Can't find all required implementations:
             'data': {
                 'unpack_size': len(activity_info),
                 'blob_size': len(blob),
+                'digest': hashlib.sha1(blob).hexdigest(),
                 'mime_type': 'application/vnd.olpc-sugar',
                 'spec': {'*-*': {'commands': {'activity': {'exec': 'true'}}, 'requires': {}}},
                 },
@@ -713,6 +715,7 @@ Can't find all required implementations:
             'seqno': 5,
             'unpack_size': len(activity_info),
             'blob_size': len(blob),
+            'digest': hashlib.sha1(blob).hexdigest(),
             'blob': blob_path,
             'mtime': int(os.stat(blob_path[:-5]).st_mtime),
             'mime_type': 'application/vnd.olpc-sugar',
@@ -756,6 +759,7 @@ Can't find all required implementations:
             'seqno': 5,
             'unpack_size': len(activity_info),
             'blob_size': len(blob),
+            'digest': hashlib.sha1(blob).hexdigest(),
             'blob': blob_path,
             'mtime': int(os.stat(blob_path[:-5]).st_mtime),
             'mime_type': 'application/vnd.olpc-sugar',
@@ -878,6 +882,7 @@ Can't find all required implementations:
             'tags': [],
             'data': {
                 'blob_size': len(blob),
+                'digest': hashlib.sha1(blob).hexdigest(),
                 'mime_type': 'application/vnd.olpc-sugar',
                 'spec': {'*-*': {'commands': {'activity': {'exec': 'true'}}, 'requires': {}}},
                 'unpack_size': len(activity_info),
@@ -904,6 +909,7 @@ Can't find all required implementations:
             'data': {
                 'blob': blob_path,
                 'blob_size': len(blob),
+                'digest': hashlib.sha1(blob).hexdigest(),
                 'mime_type': 'application/vnd.olpc-sugar',
                 'mtime': int(os.stat(blob_path[:-5]).st_mtime),
                 'seqno': 5,
@@ -943,6 +949,7 @@ Can't find all required implementations:
             'tags': [],
             'data': {
                 'blob_size': len(blob),
+                'digest': hashlib.sha1(blob).hexdigest(),
                 'unpack_size': len(activity_info),
                 'mime_type': 'application/vnd.olpc-sugar',
                 'spec': {'*-*': {'commands': {'activity': {'exec': 'true'}}, 'requires': {}}},
@@ -989,6 +996,7 @@ Can't find all required implementations:
             'tags': [],
             'data': {
                 'blob_size': len(blob),
+                'digest': hashlib.sha1(blob).hexdigest(),
                 'unpack_size': len(activity_info),
                 'mime_type': 'application/vnd.olpc-sugar',
                 'spec': {'*-*': {'commands': {'activity': {'exec': 'true'}}, 'requires': {}}},
@@ -1091,6 +1099,7 @@ Can't find all required implementations:
             'tags': [],
             'data': {
                 'blob_size': len(blob),
+                'digest': hashlib.sha1(blob).hexdigest(),
                 'unpack_size': len(activity_info),
                 'mime_type': 'application/vnd.olpc-sugar',
                 'spec': {'*-*': {'commands': {'activity': {'exec': 'false'}}, 'requires': {}}},