Import ASLO previews and comments

2 files changed, 144 insertions, 24 deletions

diff --git a/misc/aslo-sync b/misc/aslo-sync
index 6b92d55..cf90b9f 100755
--- a/misc/aslo-sync
+++ b/misc/aslo-sync
@@ -18,21 +18,19 @@
 import os
 import sys
 import time
-import shutil
 import getpass
-import gettext
-import tempfile
+import hashlib
 import traceback
+from cStringIO import StringIO
 from os.path import join
 
 import MySQLdb as mdb
 
 from sugar_network import db, client, model, toolkit
 from sugar_network.node import data_root
-from sugar_network.client.bundle import Bundle
 from sugar_network.node.slave import SlaveRoutes
 from sugar_network.node.routes import load_bundle
-from sugar_network.toolkit import util, licenses, application, spec, Option
+from sugar_network.toolkit import util, licenses, application, Option
 
 
 DOWNLOAD_URL = 'http://download.sugarlabs.org/activities'
@@ -98,6 +96,11 @@ IGNORE_VERSIONS = frozenset([
         31512,  # Bad license
         ])
 
+IGNORE_PREVIEWS = frozenset([
+    475,    # Malformed PNG
+    476,    # Malformed PNG
+    ])
+
 LICENSES_MAP = {
         'org.laptop.x2o': ['GPLv2+'],
         'org.wesnoth.Wesnoth': ['GPLv2'],
@@ -149,12 +152,6 @@ class Application(application.Application):
             self._volume.populate()
         return self._volume
 
-    @property
-    def client(self):
-        if self._client is None:
-            self._client = client.Client()
-        return self._client
-
     def epilog(self):
         if self._volume is not None:
             self._volume.close()
@@ -254,9 +251,11 @@ class Application(application.Application):
             if [i for i in IGNORE_ADDONS if i in bundle_id]:
                 continue
             try:
-                self.sync_context(addon_id, bundle_id)
+                authors = self.sync_context(addon_id, bundle_id)
                 self.sync_versions(addon_id, bundle_id)
                 self.sync_reviews(addon_id, bundle_id)
+                self.sync_previews(addon_id, bundle_id, authors)
+                self.sync_comments(addon_id, bundle_id)
             except Exception:
                 print '-- Failed to sync %s addon' % addon_id
                 traceback.print_exception(*sys.exc_info())
@@ -267,10 +266,112 @@ class Application(application.Application):
             print '-- Hide %r deleted activity' % guid
             directory.update(guid, {'layer': ['deleted']})
 
+    def sync_previews(self, addon_id, bundle_id, authors):
+        directory = self.volume['artifact']
+        items, __ = directory.find(context=bundle_id, type='preview',
+                not_layer='deleted')
+        existing = set([i.guid for i in items])
+
+        sql = """
+            SELECT
+                id,
+                created,
+                caption,
+                filedata
+            FROM
+                previews
+            WHERE
+                addon_id = %s
+            """ % addon_id
+        for guid, created, caption, data in self.sqlexec(sql):
+            if guid in IGNORE_PREVIEWS:
+                continue
+            guid = str(guid)
+            if directory.exists(guid):
+                existing.remove(guid)
+                continue
+            try:
+                preview = scale_png(data, 160, 120)
+            except Exception:
+                print '-- Failed to load %s preview for %s' % (guid, bundle_id)
+                continue
+            directory.create({
+                    'guid': guid,
+                    'ctime': int(time.mktime(created.timetuple())),
+                    'mtime': int(time.mktime(created.timetuple())),
+                    'context': bundle_id,
+                    'type': 'preview',
+                    'title': self.get_i18n_field(caption),
+                    'description': self.get_i18n_field(caption),
+                    'author': authors,
+                    'preview': {
+                        'blob': StringIO(preview),
+                        'mime_type': 'image/png',
+                        'digest': hashlib.sha1(preview).hexdigest(),
+                        },
+                    'data': {
+                        'blob': StringIO(data),
+                        'mime_type': 'image/png',
+                        'digest': hashlib.sha1(data).hexdigest(),
+                        },
+                    })
+
+        for guid in existing:
+            print '-- Hide %s %s deleted preview' % (bundle_id, guid)
+            directory.update(guid, {'layer': ['deleted']})
+
+    def sync_comments(self, addon_id, bundle_id):
+        directory = self.volume['comment']
+        items, __ = directory.find(context=bundle_id, not_layer='deleted')
+        existing = set([i.guid for i in items])
+
+        sql = """
+            SELECT
+                reviews.id,
+                reviews.created,
+                reviews.body,
+                users.email,
+                users.nickname,
+                CONCAT_WS(' ', users.firstname, users.lastname),
+                reviews.reply_to
+            FROM
+                reviews
+                INNER JOIN versions ON versions.id = reviews.version_id
+                INNER JOIN users ON users.id=reviews.user_id
+            WHERE
+                reply_to IS NOT NULL AND versions.addon_id = %s
+            """ % addon_id
+        for guid, created, content, email, nickname, fullname, reply_to \
+                in self.sqlexec(sql):
+            guid = str(guid)
+            if directory.exists(guid):
+                existing.remove(guid)
+                continue
+            if not nickname:
+                nickname = email.split('@')[0]
+            fullname = fullname.strip()
+            if not fullname:
+                fullname = nickname
+            directory.create({
+                    'guid': guid,
+                    'ctime': int(time.mktime(created.timetuple())),
+                    'mtime': int(time.mktime(created.timetuple())),
+                    'context': bundle_id,
+                    'review': str(reply_to),
+                    'message': self.get_i18n_field(content),
+                    'author': {nickname: {
+                        'order': 0, 'role': 3, 'name': fullname,
+                        }},
+                    })
+
+        for guid in existing:
+            print '-- Hide %s %s deleted comment' % (bundle_id, guid)
+            directory.update(guid, {'layer': ['deleted']})
+
     def sync_reviews(self, addon_id, bundle_id):
         directory = self.volume['review']
         items, __ = directory.find(context=bundle_id, not_layer='deleted')
-        existing_reviews = set([i.guid for i in items])
+        existing = set([i.guid for i in items])
 
         sql = """
             SELECT
@@ -293,7 +394,7 @@ class Application(application.Application):
                 in self.sqlexec(sql):
             guid = str(guid)
             if directory.exists(guid):
-                existing_reviews.remove(guid)
+                existing.remove(guid)
                 continue
             if not nickname:
                 nickname = email.split('@')[0]
@@ -313,14 +414,14 @@ class Application(application.Application):
                         }},
                     })
 
-        for guid in existing_reviews:
+        for guid in existing:
             print '-- Hide %s %s deleted review' % (bundle_id, guid)
             directory.update(guid, {'layer': ['deleted']})
 
     def sync_versions(self, addon_id, bundle_id):
         directory = self.volume['implementation']
         items, __ = directory.find(context=bundle_id, not_layer='deleted')
-        existing_versions = set([i.guid for i in items])
+        existing = set([i.guid for i in items])
 
         sql = """
             SELECT
@@ -366,7 +467,7 @@ class Application(application.Application):
                 continue
 
             try:
-                parsed_version = util.parse_version(version)
+                util.parse_version(version)
             except Exception, error:
                 print '-- Cannot parse %r version for %r[%s]: %s' % \
                         (version, filename, version_id, error)
@@ -413,17 +514,18 @@ class Application(application.Application):
 
             if directory.exists(version_id):
                 if set(directory.get(version_id).layer) != set(layers) or \
-                        version_id not in existing_versions:
+                        version_id not in existing:
                     directory.update(version_id, {'layer': layers})
-                if version_id in existing_versions:
-                    existing_versions.remove(version_id)
+                if version_id in existing:
+                    existing.remove(version_id)
                 continue
 
             bundle_path = join(ACTIVITIES_PATH, str(addon_id), filename)
             try:
                 with load_bundle(
                         self.volume, Request(self.volume, {
-                            'requires': 'sugar>=%s<=%s' % (sugar_min, sugar_max),
+                            'requires': 'sugar>=%s<=%s' %
+                                    (sugar_min, sugar_max),
                             'license': alicense,
                             }),
                         bundle_path) as impl:
@@ -446,7 +548,7 @@ class Application(application.Application):
             else:
                 print '-- Sync %r' % filename
 
-        for guid in existing_versions:
+        for guid in existing:
             print '-- Hide %s %s deleted version' % (bundle_id, guid)
             directory.update(guid, {'layer': ['deleted']})
 
@@ -540,6 +642,8 @@ class Application(application.Application):
                 })
         print '-- Sync %r activity' % bundle_id
 
+        return authors
+
     def parse_license(self, alicense):
         for good in licenses.GOOD_LICENSES:
             if not alicense or good in ['ec']:
@@ -602,6 +706,22 @@ class Request(dict):
             self._volume[resource].update(guid, content)
 
 
+def scale_png(data, w, h):
+    with toolkit.NamedTemporaryFile() as src:
+        src.write(data)
+        src.flush()
+        with toolkit.NamedTemporaryFile() as dst:
+            toolkit.assert_call(['convert',
+                '-thumbnail', '%sx%s' % (w, h),
+                '-background', 'transparent',
+                '-gravity', 'center',
+                '-extent', '%sx%s' % (w, h),
+                src.name, dst.name,
+                ])
+            with file(dst.name, 'rb') as f:
+                return f.read()
+
+
 mysql_server = Option(
         'MySQL server',
         default='localhost', name='mysql_server')
diff --git a/sugar-network b/sugar-network
index 224fe77..6f83100 100755
--- a/sugar-network
+++ b/sugar-network
@@ -241,7 +241,7 @@ class Application(application.Application):
                         chunk = result.read(BUFFER_SIZE)
                         if not chunk:
                             break
-                        self._print(chunk, '\n')
+                        self._print(chunk)
             else:
                 self._print(result, '\n')
         finally:
@@ -309,7 +309,7 @@ class Application(application.Application):
 
     def _print(self, *data):
         if not quiet.value:
-            print ''.join(data),
+            sys.stdout.write(''.join(data))
 
 
 # Let toolkit.http work in concurrence