Node command to regerate package resolves/presolves

7 files changed, 109 insertions, 105 deletions

diff --git a/sugar_network/client/injector.py b/sugar_network/client/injector.py
index 69dc06a..fcd87ea 100644
--- a/sugar_network/client/injector.py
+++ b/sugar_network/client/injector.py
@@ -24,12 +24,12 @@ import logging
 from os.path import exists, join
 
 from sugar_network import toolkit
-from sugar_network.client import packagekit, journal, profile_path
+from sugar_network.client import journal, profile_path
 from sugar_network.toolkit.spec import format_version
 from sugar_network.toolkit.bundle import Bundle
 from sugar_network.toolkit.coroutine import this
 from sugar_network.toolkit import lsb_release, coroutine, i18n, pylru, http
-from sugar_network.toolkit import enforce
+from sugar_network.toolkit import packagekit, enforce
 
 
 _PREEMPTIVE_POOL_SIZE = 256
diff --git a/sugar_network/db/blobs.py b/sugar_network/db/blobs.py
index f283cb2..0566e4f 100644
--- a/sugar_network/db/blobs.py
+++ b/sugar_network/db/blobs.py
@@ -243,7 +243,7 @@ class Blobs(object):
         else:
             path = self._blob_path(patch.digest)
         if not patch.size:
-            self._delete(path.digest, path, seqno)
+            self._delete(patch.digest, path, seqno)
             return
         if not exists(dirname(path)):
             os.makedirs(dirname(path))
diff --git a/sugar_network/db/resource.py b/sugar_network/db/resource.py
index 35ab29b..1bf46c6 100644
--- a/sugar_network/db/resource.py
+++ b/sugar_network/db/resource.py
@@ -208,7 +208,11 @@ class Resource(object):
             patch[prop] = value
         return patch
 
-    def post(self, prop, value, **meta):
+    def post(self, prop, value=None, **meta):
+        if value is None:
+            if prop not in self.posts:
+                return
+            value = self.posts[prop]
         prop = self.metadata[prop]
         if prop.on_set is not None:
             value = prop.on_set(self, value)
diff --git a/sugar_network/node/master.py b/sugar_network/node/master.py
index d51d85b..d5f3f70 100644
--- a/sugar_network/node/master.py
+++ b/sugar_network/node/master.py
@@ -19,9 +19,9 @@ from urlparse import urlsplit
 from sugar_network import toolkit
 from sugar_network.model.post import Post
 from sugar_network.model.report import Report
-from sugar_network.node import obs, model
+from sugar_network.node import model
 from sugar_network.node.routes import NodeRoutes
-from sugar_network.toolkit.router import route, ACL
+from sugar_network.toolkit.router import route
 from sugar_network.toolkit.coroutine import this
 from sugar_network.toolkit import http, packets, pylru, ranges, enforce
 
@@ -55,13 +55,6 @@ class MasterRoutes(NodeRoutes):
         return packets.encode(reply, limit=accept_length,
                 header={'from': self.guid}, on_complete=this.cookie.clear)
 
-    @route('PUT', ['context', None], cmd='presolve',
-            acl=ACL.AUTH, mime_type='application/json')
-    def presolve(self):
-        aliases = this.volume['context'].get(this.request.guid)['aliases']
-        enforce(aliases, http.BadRequest, 'Nothing to presolve')
-        return obs.presolve(None, aliases, this.volume.blobs.path('packages'))
-
     def status(self):
         result = NodeRoutes.status(self)
         result['mode'] = 'master'
diff --git a/sugar_network/node/model.py b/sugar_network/node/model.py
index 380963f..545993f 100644
--- a/sugar_network/node/model.py
+++ b/sugar_network/node/model.py
@@ -29,18 +29,18 @@ from sugar_network.model import ICON_SIZE, LOGO_SIZE
 from sugar_network.node import obs
 from sugar_network.node.auth import Principal
 from sugar_network.toolkit.router import ACL, File, Request, Response
-from sugar_network.toolkit.coroutine import Queue, this
+from sugar_network.toolkit.coroutine import Queue, Empty, this
 from sugar_network.toolkit.spec import EMPTY_LICENSE, ensure_version
 from sugar_network.toolkit.spec import parse_requires, parse_version
 from sugar_network.toolkit.bundle import Bundle
-from sugar_network.toolkit import sat, http, i18n, ranges, packets
+from sugar_network.toolkit import sat, http, i18n, ranges, packets, packagekit
 from sugar_network.toolkit import svg_to_png, enforce
 
 
 BATCH_SUFFIX = '.meta'
 
 _logger = logging.getLogger('node.model')
-_presolve_queue = None
+_presolve_queue = Queue()
 
 
 class User(_user.User):
@@ -64,9 +64,11 @@ class _Release(object):
             return value.guid, value
         doc = this.volume['context'][this.request.guid]
         if 'package' in doc['type']:
+            enforce(this.request.key, http.BadRequest, 'No distro in path')
             value = _ReleaseValue(self._package_subcast.typecast(value))
             value.guid = this.request.key
-            _resolve_package_alias(doc, value)
+            resolve_package(doc, this.request.key, value)
+            doc.post('releases')
             return value
         bundle = this.volume.blobs.post(value, this.request.content_type)
         __, value = load_bundle(bundle, context=this.request.guid)
@@ -336,7 +338,9 @@ def solve(volume, top_context, command=None, lsb_id=None, lsb_release=None,
         if 'package' in context['type']:
             pkg_lst = None
             pkg_ver = []
-            pkg = releases.get('resolves', {}).get(lsb_distro)
+            pkg = None
+            if 'resolves' in releases:
+                pkg = releases['resolves']['value'].get(lsb_distro)
             if pkg:
                 pkg_ver = pkg['version']
                 pkg_lst = pkg['packages']
@@ -418,12 +422,61 @@ def solve(volume, top_context, command=None, lsb_id=None, lsb_release=None,
     return solution
 
 
-def presolve(presolve_path):
-    global _presolve_queue
-    _presolve_queue = Queue()
+def resolve_package(doc, distro, alias):
+    enforce(alias.get('binary'), http.BadRequest, 'No binary aliases')
 
-    for repo_name, pkgs in _presolve_queue:
-        obs.presolve(repo_name, pkgs, presolve_path)
+    if distro == '*':
+        lsb_id = None
+        lsb_release = None
+    elif '-' in distro:
+        lsb_id, lsb_release = distro.split('-', 1)
+    else:
+        lsb_id = distro
+        lsb_release = None
+    releases = doc['releases']
+    resolves = releases['resolves']['value'] if 'resolves' in releases else {}
+    to_presolve = []
+
+    for repo in obs.get_repos():
+        if lsb_id and lsb_id != repo['lsb_id'] or \
+                lsb_release and lsb_release != repo['lsb_release']:
+            continue
+        # Make sure there are no alias overrides
+        if not lsb_id and repo['lsb_id'] in releases or \
+                not lsb_release and repo['name'] in releases:
+            continue
+        pkgs = sum([alias.get(i, []) for i in ('binary', 'devel')], [])
+        version = None
+        try:
+            for arch in repo['arches']:
+                version = obs.resolve(repo['name'], arch, pkgs)['version']
+        except Exception, error:
+            _logger.warning('Failed to resolve %r on %s',
+                    pkgs, repo['name'])
+            resolve = {'status': str(error)}
+        else:
+            to_presolve.append((repo['name'], pkgs))
+            version = packagekit.cleanup_distro_version(version)
+            resolve = {
+                    'version': parse_version(version),
+                    'packages': pkgs,
+                    'status': 'success',
+                    }
+        resolves.setdefault(repo['name'], {}).update(resolve)
+
+    if to_presolve and _presolve_queue is not None:
+        _presolve_queue.put(to_presolve)
+    doc.posts.setdefault('releases', {})['resolves'] = {'value': resolves}
+
+
+def presolve(presolve_path, **pop_kwargs):
+    while True:
+        try:
+            value = _presolve_queue.get(**pop_kwargs)
+        except Empty:
+            break
+        for repo_name, pkgs in value:
+            obs.presolve(repo_name, pkgs, presolve_path)
 
 
 def load_bundle(blob, context=None, initial=False, extra_deps=None,
@@ -624,54 +677,6 @@ def _load_context_metadata(bundle, spec):
     return result, icon_svg
 
 
-def _resolve_package_alias(doc, value):
-    enforce(value.get('binary'), http.BadRequest, 'No binary aliases')
-
-    distro = this.request.key
-    enforce(distro, http.BadRequest, 'No distro in path')
-    if distro == '*':
-        lsb_id = None
-        lsb_release = None
-    elif '-' in this.request.key:
-        lsb_id, lsb_release = distro.split('-', 1)
-    else:
-        lsb_id = distro
-        lsb_release = None
-    releases = doc['releases']
-    resolves = releases.get('resolves') or {}
-    to_presolve = []
-
-    for repo in obs.get_repos():
-        if lsb_id and lsb_id != repo['lsb_id'] or \
-                lsb_release and lsb_release != repo['lsb_release']:
-            continue
-        # Make sure there are no alias overrides
-        if not lsb_id and repo['lsb_id'] in releases or \
-                not lsb_release and repo['name'] in releases:
-            continue
-        pkgs = sum([value.get(i, []) for i in ('binary', 'devel')], [])
-        version = None
-        try:
-            for arch in repo['arches']:
-                version = obs.resolve(repo['name'], arch, pkgs)['version']
-        except Exception, error:
-            _logger.warning('Failed to resolve %r on %s',
-                    pkgs, repo['name'])
-            resolve = {'status': str(error)}
-        else:
-            to_presolve.append((repo['name'], pkgs))
-            resolve = {
-                    'version': parse_version(version),
-                    'packages': pkgs,
-                    'status': 'success',
-                    }
-        resolves.setdefault(repo['name'], {}).update(resolve)
-
-    if to_presolve and _presolve_queue is not None:
-        _presolve_queue.put(to_presolve)
-    doc.post('releases', {'resolves': resolves})
-
-
 def _generate_icons(svg, props):
     blobs = this.volume.blobs
     props['artefact_icon'] = \
diff --git a/sugar_network/node/obs.py b/sugar_network/node/obs.py
index 0c68a6e..81afc62 100644
--- a/sugar_network/node/obs.py
+++ b/sugar_network/node/obs.py
@@ -56,6 +56,9 @@ def resolve(repo, arch, packages):
 
 
 def presolve(repo_name, packages, dst_path):
+    # TODO The only repo is OLPC which is Fedora-14 based
+    if repo_name != 'Fedora-14':
+        return
     for repo in _get_repos(obs_presolve_project.value):
         dst_dir = join(dst_path, 'packages',
                 obs_presolve_project.value, repo['name'])
@@ -68,7 +71,7 @@ def presolve(repo_name, packages, dst_path):
                 for repo_arch in repo['arches']:
                     response = _request('GET', ['resolve'], params={
                         'project': obs_presolve_project.value,
-                        'repository': '%(lsb_id)s-%(lsb_release)s' % repo,
+                        'repository': repo['name'],
                         'arch': repo_arch,
                         'package': package,
                         'withdeps': '1',
@@ -129,14 +132,13 @@ def _get_repos(project):
 
     repos = _repos[project] = []
     for repo in _request('GET', ['build', project]).findall('entry'):
-        repo = repo.get('name')
-        arches = _request('GET', ['build', project, repo])
-        lsb_id, lsb_release = repo.split('-', 1)
-        repos.append({
-            'lsb_id': lsb_id,
-            'lsb_release': lsb_release,
-            'name': repo,
-            'arches': [i.get('name') for i in arches.findall('entry')],
-            })
+        name = repo.get('name')
+        arches = _request('GET', ['build', project, name])
+        info = {'name': name,
+                'arches': [i.get('name') for i in arches.findall('entry')],
+                }
+        if '-' in name:
+            info['lsb_id'], info['lsb_release'] = name.split('-', 1)
+        repos.append(info)
 
     return repos
diff --git a/sugar_network/client/packagekit.py b/sugar_network/toolkit/packagekit.py
index 68772e8..6003486 100644
--- a/sugar_network/client/packagekit.py
+++ b/sugar_network/toolkit/packagekit.py
@@ -54,6 +54,29 @@ def install(names):
                 resp.error_details, resp.error_code)
 
 
+def cleanup_distro_version(version):
+    if ':' in version:
+        # Skip 'epoch'
+        version = version.split(':', 1)[1]
+    version = version.replace('_', '-')
+    if '~' in version:
+        version, suffix = version.split('~', 1)
+        if suffix.startswith('pre'):
+            suffix = suffix[3:]
+        suffix = '-pre' + (cleanup_distro_version(suffix) or '')
+    else:
+        suffix = ''
+    match = _VERSION_RE.match(version)
+    if match:
+        major, version, revision = match.groups()
+        if major is not None:
+            version = major[:-1].rstrip('.') + '.' + version
+        if revision is not None:
+            version = '%s-%s' % (version, revision[2:])
+        return version + suffix
+    return None
+
+
 def _resolve(names):
     result = {}
 
@@ -106,7 +129,7 @@ def _pk(result, op, *args):
 
     def Package_cb(status, pk_id, summary):
         package_name, version, arch, __ = pk_id.split(';')
-        clean_version = _cleanup_distro_version(version)
+        clean_version = cleanup_distro_version(version)
         if not clean_version:
             _logger.warn('Cannot parse distribution version "%s" '
                     'for package "%s"', version, package_name)
@@ -159,29 +182,6 @@ def _canonicalize_machine(arch):
         return 'i686'
 
 
-def _cleanup_distro_version(version):
-    if ':' in version:
-        # Skip 'epoch'
-        version = version.split(':', 1)[1]
-    version = version.replace('_', '-')
-    if '~' in version:
-        version, suffix = version.split('~', 1)
-        if suffix.startswith('pre'):
-            suffix = suffix[3:]
-        suffix = '-pre' + (_cleanup_distro_version(suffix) or '')
-    else:
-        suffix = ''
-    match = _VERSION_RE.match(version)
-    if match:
-        major, version, revision = match.groups()
-        if major is not None:
-            version = major[:-1].rstrip('.') + '.' + version
-        if revision is not None:
-            version = '%s-%s' % (version, revision[2:])
-        return version + suffix
-    return None
-
-
 _DOTTED_RE = r'[0-9]+(?:\.[0-9]+)*'
 # Matche a version number that would be a valid version without modification
 _RELEASE_RE = '(?:%s)(?:-(?:pre|rc|post|)(?:%s))*' % (_DOTTED_RE, _DOTTED_RE)