Replace activity updater with microformat compatible one

This patch replaces the Sugar activity updater with one, that supports
activity microformats.

+ The updater now allows installation of new activities which were not
previously installed.

+ The updater uses the optional olpc-activity-name and
olpc-activity-size
tags.

   - If olpc-activity-name is not present, the activity name is derived
     from the bundle id. For example, org.Sugarlabs.RecordActivity
     will be listed as Record.

   - If olpc-activity-size is not present, an additional
     http request is made to ascertain the size of the
     bundle.

+ If the size returned as zero, the bundle is removed from the list of
 those which may be updated.

+ To install new bundles, a metabundle class has been created, which
acts as an empty structure.

Co-Authored by Anish Mangal <anish@sugarlabs.org>
Co-Authored by Akash Gangil <akashg1611@gmail.com>

Signed-off-by: Anish Mangal <anish@sugarlabs.org>

6 files changed, 285 insertions, 201 deletions

diff --git a/data/sugar.schemas.in b/data/sugar.schemas.in
index bcb7903..670226a 100644
--- a/data/sugar.schemas.in
+++ b/data/sugar.schemas.in
@@ -117,6 +117,18 @@
     </schema>
 
     <schema>
+      <key>/schemas/desktop/sugar/updater_url</key>
+      <applyto>/desktop/sugar/updater_url</applyto>
+      <owner>sugar</owner>
+      <type>string</type>
+      <default>http://activities-testing.sugarlabs.org/services/micro-format.php?collection_nickname=fructose</default>
+      <locale name="C">
+        <short>Activity updater URL.</short>
+        <long>This key contains the URL which the microformat compatible activity updater will search for activity updates.</long>
+      </locale>
+    </schema>
+
+    <schema>
       <key>/schemas/desktop/sugar/backup_url</key>
       <applyto>/desktop/sugar/backup_url</applyto>
       <owner>sugar</owner>
diff --git a/extensions/cpsection/updater/backends/Makefile.am b/extensions/cpsection/updater/backends/Makefile.am
index e280a07..e9c1284 100644
--- a/extensions/cpsection/updater/backends/Makefile.am
+++ b/extensions/cpsection/updater/backends/Makefile.am
@@ -1,5 +1,5 @@
 sugardir = $(pkgdatadir)/extensions/cpsection/updater/backends
 
 sugar_PYTHON = 		\
-	aslo.py		\
+	microformat.py	\
 	__init__.py
diff --git a/extensions/cpsection/updater/backends/aslo.py b/extensions/cpsection/updater/backends/aslo.py
deleted file mode 100644
index 6504e9e..0000000
--- a/extensions/cpsection/updater/backends/aslo.py
+++ /dev/null
@@ -1,164 +0,0 @@
-#!/usr/bin/python
-# Copyright (C) 2009, Sugar Labs
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
-
-"""Activity information microformat parser.
-
-Activity information is embedded in HTML/XHTML/XML pages using a
-Resource Description Framework (RDF) http://www.w3.org/RDF/ .
-
-An example::
-
-<?xml version="1.0" encoding="UTF-8"?>
-<RDF:RDF xmlns:RDF="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-        xmlns:em="http://www.mozilla.org/2004/em-rdf#">
-<RDF:Description about="urn:mozilla:extension:bounce">
-    <em:updates>
-        <RDF:Seq>
-            <RDF:li resource="urn:mozilla:extension:bounce:7"/>
-        </RDF:Seq>
-    </em:updates>
-</RDF:Description>
-
-<RDF:Description about="urn:mozilla:extension:bounce:7">
-    <em:version>7</em:version>
-    <em:targetApplication>
-        <RDF:Description>
-            <em:id>{3ca105e0-2280-4897-99a0-c277d1b733d2}</em:id>
-            <em:minVersion>0.82</em:minVersion>
-            <em:maxVersion>0.84</em:maxVersion>
-            <em:updateLink>http://foo.xo</em:updateLink>
-            <em:updateSize>7</em:updateSize>
-            <em:updateHash>sha256:816a7c43b4f1ea4769c61c03ea4..</em:updateHash>
-        </RDF:Description>
-    </em:targetApplication>
-</RDF:Description></RDF:RDF>
-"""
-
-import logging
-from xml.etree.ElementTree import XML
-import traceback
-
-import gio
-
-from sugar.bundle.bundleversion import NormalizedVersion
-from sugar.bundle.bundleversion import InvalidVersionError
-
-from jarabe import config
-
-_FIND_DESCRIPTION = \
-        './/{http://www.w3.org/1999/02/22-rdf-syntax-ns#}Description'
-_FIND_VERSION = './/{http://www.mozilla.org/2004/em-rdf#}version'
-_FIND_LINK = './/{http://www.mozilla.org/2004/em-rdf#}updateLink'
-_FIND_SIZE = './/{http://www.mozilla.org/2004/em-rdf#}updateSize'
-
-_UPDATE_PATH = 'http://activities.sugarlabs.org/services/update-aslo.php'
-
-_fetcher = None
-
-
-class _UpdateFetcher(object):
-
-    _CHUNK_SIZE = 10240
-
-    def __init__(self, bundle, completion_cb):
-        # ASLO knows only about stable SP releases
-        major, minor = config.version.split('.')[0:2]
-        sp_version = '%s.%s' % (major, int(minor) + int(minor) % 2)
-
-        url = '%s?id=%s&appVersion=%s' % \
-                (_UPDATE_PATH, bundle.get_bundle_id(), sp_version)
-
-        logging.debug('Fetch %s', url)
-
-        self._completion_cb = completion_cb
-        self._file = gio.File(url)
-        self._stream = None
-        self._xml_data = ''
-        self._bundle = bundle
-
-        self._file.read_async(self.__file_read_async_cb)
-
-    def __file_read_async_cb(self, gfile, result):
-        try:
-            self._stream = self._file.read_finish(result)
-        except:
-            global _fetcher
-            _fetcher = None
-            self._completion_cb(None, None, None, None, traceback.format_exc())
-            return
-
-        self._stream.read_async(self._CHUNK_SIZE, self.__stream_read_async_cb)
-
-    def __stream_read_async_cb(self, stream, result):
-        xml_data = self._stream.read_finish(result)
-        if xml_data is None:
-            global _fetcher
-            _fetcher = None
-            self._completion_cb(self._bundle, None, None, None,
-                    'Error reading update information for %s from '
-                    'server.' % self._bundle.get_bundle_id())
-            return
-        elif not xml_data:
-            self._process_result()
-        else:
-            self._xml_data += xml_data
-            self._stream.read_async(self._CHUNK_SIZE,
-                                    self.__stream_read_async_cb)
-
-    def _process_result(self):
-        document = XML(self._xml_data)
-
-        if document.find(_FIND_DESCRIPTION) is None:
-            logging.debug('Bundle %s not available in the server for the '
-                'version %s', self._bundle.get_bundle_id(), config.version)
-            version = None
-            link = None
-            size = None
-        else:
-            try:
-                version = NormalizedVersion(document.find(_FIND_VERSION).text)
-            except InvalidVersionError:
-                logging.exception('Exception occured while parsing version')
-                version = '0'
-
-            link = document.find(_FIND_LINK).text
-
-            try:
-                size = long(document.find(_FIND_SIZE).text) * 1024
-            except ValueError:
-                logging.exception('Exception occured while parsing size')
-                size = 0
-
-        global _fetcher
-        _fetcher = None
-        self._completion_cb(self._bundle, version, link, size, None)
-
-
-def fetch_update_info(bundle, completion_cb):
-    """Queries the server for a newer version of the ActivityBundle.
-
-       completion_cb receives bundle, version, link, size and possibly an error
-       message:
-
-       def completion_cb(bundle, version, link, size, error_message):
-    """
-    global _fetcher
-
-    if _fetcher is not None:
-        raise RuntimeError('Multiple simultaneous requests are not supported')
-
-    _fetcher = _UpdateFetcher(bundle, completion_cb)
diff --git a/extensions/cpsection/updater/backends/microformat.py b/extensions/cpsection/updater/backends/microformat.py
new file mode 100644
index 0000000..97499aa
--- /dev/null
+++ b/extensions/cpsection/updater/backends/microformat.py
@@ -0,0 +1,203 @@
+#!/usr/bin/python
+#
+# Copyright (C) 2011, Anish Mangal <anish@sugarlabs.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+
+import logging
+from HTMLParser import HTMLParser
+import urllib
+import re
+
+import gio
+import gobject
+import gconf
+
+from jarabe import config
+
+client = gconf.client_get_default()
+_UPDATE_PATH = client.get_string('/desktop/sugar/updater_url')
+_ACTIVITIES_LIST = {}
+ACTION_CHECKING = 0
+ACTION_UPDATING = 1
+ACTION_DOWNLOADING = 2
+
+class MicroformatParser(HTMLParser):
+
+    def __init__(self, data, completion_cb):
+        HTMLParser.__init__(self)
+        self.reset()
+        self._data_to_parse = data
+        self._activity_id = ''
+        self._activity_url = ''
+        self._activity_version = ''
+        self._activity_size = 1
+        self._activity_name = ''
+        self._inside_activity_block = False
+        self._inside_activity_version = False
+        self._inside_activity_id = False
+        self._inside_activity_url = False
+        self._inside_activity_size = False
+        self._inside_activity_name = False
+        self._activity_block_tag = ''
+        self._completion_cb = completion_cb
+
+    def parse(self):
+        self.feed(self._data_to_parse)
+
+    def handle_endtag(self, tag):
+        if tag == self._activity_block_tag and self._inside_activity_block:
+            self._inside_activity_block = False
+
+            _ACTIVITIES_LIST[self._activity_id] = \
+                    {'version':self._activity_version,
+                            'url':self._activity_url,
+                            'size':self._activity_size,
+                            'name':self._activity_name}
+
+        elif tag == 'a':
+            if self._inside_activity_url:
+                self._inside_activity_url = False
+
+        elif tag == 'body':
+            num_bundles = len(_ACTIVITIES_LIST)
+            progress = num_bundles
+            for bundle, info in _ACTIVITIES_LIST.items():
+                progress = progress + 1
+                if _ACTIVITIES_LIST[bundle]['size'] == 1:
+                    try:
+                        _ACTIVITIES_LIST[bundle]['size'] = \
+                            gio.File(_ACTIVITIES_LIST[bundle]['url']).\
+                            query_info('*').get_size()
+
+                    except Exception, e:
+                        logging.exception(e)
+
+                    if _ACTIVITIES_LIST[bundle]['size'] == 0:
+                        logging.error('Size of activity %s reported as '
+                                '0 bytes. Excluding from update list' % bundle)
+                        del _ACTIVITIES_LIST[bundle]
+
+                    elif _ACTIVITIES_LIST[bundle]['name'] == '':
+                        # Do some regex magic to get the 'probable'
+                        # activity name.
+                        activity_name = re.split('\.',
+                                bundle)[-1]
+                        activity_name = re.sub('^[\s|\t]*', '',
+                                activity_name)
+                        activity_name = re.sub('[\s|\t]*$', '',
+                                activity_name)
+                        activity_name = re.sub('[A|a]ctivity$', '',
+                                activity_name)
+                        _ACTIVITIES_LIST[bundle]['name'] = \
+                                activity_name
+
+            self._completion_cb(_ACTIVITIES_LIST, None)
+
+    def handle_starttag(self, tag, attrs):
+        for attribute, value in attrs:
+            if value == 'olpc-activity-info':
+                self._inside_activity_block = True
+                self._activity_block_tag = tag
+
+        if tag == 'span':
+            for attribute, value in attrs:
+                if value == 'olpc-activity-id':
+                    self._inside_activity_id = True
+                elif value == 'olpc-activity-version':
+                    self._inside_activity_version = True
+                elif value == 'olpc-activity-name':
+                    self._inside_activity_name = True
+                elif value == 'olpc-activity-size':
+                    self._inside_activity_size = True
+                elif value == 'olpc-activity-url':
+                    self._inside_activity_url = True
+
+        elif tag == 'a':
+            if self._inside_activity_url:
+                for attribute, value in attrs:
+                    if attribute == 'href':
+                        self._activity_url = value
+
+    def handle_data(self, data):
+        if self._inside_activity_version:
+            self._activity_version = int(data)
+            self._inside_activity_version = False
+
+        elif self._inside_activity_id:
+            self._activity_id = data
+            self._inside_activity_id = False
+
+        elif self._inside_activity_name:
+            self._activity_name = data
+            self._inside_activity_name = False
+
+        elif self._inside_activity_size:
+            self._activity_size = int(data)
+            self._inside_activity_size = False
+
+class _UpdateFetcher(gobject.GObject):
+
+    __gsignals__ = {
+        'progress': (gobject.SIGNAL_RUN_FIRST,
+                     gobject.TYPE_NONE,
+                     ([int, str, float, int])),
+    }
+
+    def __init__(self, completion_cb):
+        gobject.GObject.__init__(self)
+        # ASLO knows only about stable SP releases
+        major, minor = config.version.split('.')[0:2]
+        sp_version = '%s.%s' % (major, int(minor) + int(minor) % 2)
+        self._data = ''
+        self._completion_cb = completion_cb
+
+    def download_bundle_updates(self):
+        self.emit('progress', ACTION_CHECKING, 'Fetching update '
+                'information', 1, 3)
+        self._url = _UPDATE_PATH
+        self._file = gio.File(self._url)
+        logging.debug('Fetch %s', self._url)
+        self._file.read_async(self.__read_async_cb)
+
+    def __read_async_cb(self, gfile, result):
+        try:
+            stream = gfile.read_finish(result)
+        except gio.Error, e:
+            self.stop()
+            logging.exception('Error while fetching content from %s' %
+                    self._url)
+            return
+        stream.read_async(4096, self.__stream_read_cb)
+
+    def __stream_read_cb(self, stream, result):
+        data = stream.read_finish(result)
+        if not data:
+            self._data_finished()
+            return
+        self._data_read(data)
+        stream.read_async(4096, self.__stream_read_cb)
+
+    def _data_read(self, data):
+        self._data += data
+
+    def read_finish(self):
+        pass
+
+    def _data_finished(self):
+        self.emit('progress', ACTION_CHECKING, 'Fetching update '
+                'information', 2, 3)
+        parser = MicroformatParser(self._data, self._completion_cb)
+        gobject.idle_add(parser.parse)
diff --git a/extensions/cpsection/updater/model.py b/extensions/cpsection/updater/model.py
index 7ea445f..d7fd528 100755
--- a/extensions/cpsection/updater/model.py
+++ b/extensions/cpsection/updater/model.py
@@ -37,8 +37,26 @@ from sugar.bundle.bundleversion import NormalizedVersion
 
 from jarabe.model import bundleregistry
 
-from backends import aslo
+from backends import microformat
 
+class MetaBundle():
+
+    def __init__(self, bundle_id, version, name):
+        self._bundle_id = bundle_id
+        self._version = version
+        self._name = name
+
+    def get_name(self):
+        return self._name
+
+    def get_bundle_id(self):
+        return self._bundle_id
+
+    def get_icon(self):
+        pass
+
+    def get_activity_version(self):
+        return self._version
 
 class UpdateModel(gobject.GObject):
     __gtype_name__ = 'SugarUpdateModel'
@@ -63,45 +81,51 @@ class UpdateModel(gobject.GObject):
         self._downloader = None
         self._cancelling = False
 
+    def __progress_cb(self, model, action, description, current, total):
+        self.emit('progress', action, description, current, total)
+
     def check_updates(self):
         self.updates = []
+        self._current_bundles = {}
+        for bundle in bundleregistry.get_registry():
+            self._current_bundles[bundle.get_bundle_id()] =\
+                    {'version':bundle.get_activity_version(),
+                            'bundle': bundle}
         self._bundles_to_check = list(bundleregistry.get_registry())
-        self._check_next_update()
-
-    def _check_next_update(self):
-        total = len(bundleregistry.get_registry())
-        current = total - len(self._bundles_to_check)
+        self._fetcher = microformat._UpdateFetcher(self.__bundle_info_fetched_cb)
+        self._fetcher.connect('progress', self.__progress_cb)
+        gobject.idle_add(self._fetcher.download_bundle_updates)
 
-        if not self._bundles_to_check:
-            return False
-
-        bundle = self._bundles_to_check.pop()
-        self.emit('progress', UpdateModel.ACTION_CHECKING, bundle.get_name(),
-                  current, total)
-
-        aslo.fetch_update_info(bundle, self.__check_completed_cb)
-
-    def __check_completed_cb(self, bundle, version, link, size, error_message):
+    def __bundle_info_fetched_cb(self, new_bundles, error_message):
         if error_message is not None:
             logging.error('Error getting update information from server:\n'
                           '%s' % error_message)
 
-        if version is not None and \
-                version > NormalizedVersion(bundle.get_activity_version()):
-            self.updates.append(BundleUpdate(bundle, version, link, size))
-
         if self._cancelling:
             self._cancel_checking()
-        elif self._bundles_to_check:
-            gobject.idle_add(self._check_next_update)
         else:
-            total = len(bundleregistry.get_registry())
-            if bundle is None:
-                name = ''
-            else:
-                name = bundle.get_name()
-            self.emit('progress', UpdateModel.ACTION_CHECKING, name, total,
-                      total)
+            for bundle_id, info in new_bundles.items():
+                if bundle_id in self._current_bundles:
+                    if new_bundles[bundle_id]['version'] >\
+                        self._current_bundles[bundle_id]['version']:
+                        self.updates.append(BundleUpdate(
+                            self._current_bundles[bundle_id]['bundle'],
+                            new_bundles[bundle_id]['version'],
+                            new_bundles[bundle_id]['url'],
+                            new_bundles[bundle_id]['size'],
+                            'update'))
+                else:
+                    bundle = MetaBundle(bundle_id,
+                            new_bundles[bundle_id]['version'],
+                            new_bundles[bundle_id]['name'])
+                    self.updates.append(BundleUpdate(bundle,
+                        new_bundles[bundle_id]['version'],
+                        new_bundles[bundle_id]['url'],
+                        new_bundles[bundle_id]['size'],
+                        'new'))
+
+        self.emit('progress', UpdateModel.ACTION_CHECKING, 'Fetching update '
+                'information', 3, 3)
 
     def update(self, bundle_ids):
         self._bundles_to_update = []
@@ -226,12 +250,15 @@ class UpdateModel(gobject.GObject):
 
 class BundleUpdate(object):
 
-    def __init__(self, bundle, version, link, size):
+    def __init__(self, bundle, version, link, size, package_type = None):
         self.bundle = bundle
         self.version = version
         self.link = link
         self.size = size
 
+        # Specify whether installing a new bundle or updating an
+        # existing one
+        self.package_type = package_type
 
 class _Downloader(gobject.GObject):
     _CHUNK_SIZE = 10240  # 10K
diff --git a/extensions/cpsection/updater/view.py b/extensions/cpsection/updater/view.py
index 814658f..30875e4 100644
--- a/extensions/cpsection/updater/view.py
+++ b/extensions/cpsection/updater/view.py
@@ -122,7 +122,7 @@ class ActivityUpdater(SectionView):
             return
 
         if action == UpdateModel.ACTION_CHECKING:
-            message = _('Checking %s...') % bundle_name
+            message = _('%s...') % bundle_name
         elif action == UpdateModel.ACTION_DOWNLOADING:
             message = _('Downloading %s...') % bundle_name
         elif action == UpdateModel.ACTION_UPDATING:
@@ -361,11 +361,17 @@ class UpdateListModel(gtk.ListStore):
             row[self.SELECTED] = True
             row[self.ICON_FILE_NAME] = bundle_update.bundle.get_icon()
 
-            details = _('From version %(current)s to %(new)s (Size: %(size)s)')
-            details = details % \
-                    {'current': bundle_update.bundle.get_activity_version(),
-                     'new': bundle_update.version,
-                     'size': _format_size(bundle_update.size)}
+            if bundle_update.package_type == 'update':
+                details = _('From version %(current)d to %(new)s (Size: %(size)s)')
+                details = details % \
+                        {'current': bundle_update.bundle.get_activity_version(),
+                         'new': bundle_update.version,
+                         'size': _format_size(bundle_update.size)}
+            elif bundle_update.package_type == 'new':
+                details = _('Install new activity version %(new)s (Size: %(size)s)')
+                details = details % \
+                        {'new': bundle_update.version,
+                         'size': _format_size(bundle_update.size)}
 
             row[self.DESCRIPTION] = '<b>%s</b>\n%s' % \
                     (bundle_update.bundle.get_name(), details)