Fix file-less blobs streaming

3 files changed, 28 insertions, 26 deletions

diff --git a/sugar_network/db/blobs.py b/sugar_network/db/blobs.py
index cc0944f..6426341 100644
--- a/sugar_network/db/blobs.py
+++ b/sugar_network/db/blobs.py
@@ -145,11 +145,13 @@ class Blobs(object):
 
         _logger.debug('Post %r file', path)
 
+        if not exists(path):
+            path = None
         return File(path, digest, meta)
 
     def update(self, path, meta):
         path = self.path(path)
-        enforce(exists(path), http.NotFound, 'No such blob')
+        enforce(exists(path + _META_SUFFIX), http.NotFound, 'No such blob')
         orig_meta = _read_meta(path)
         orig_meta.update(meta)
         _write_meta(path, orig_meta, None)
@@ -157,7 +159,10 @@ class Blobs(object):
     def get(self, digest):
         path = self.path(digest)
         if exists(path + _META_SUFFIX):
-            return File(path, digest, _read_meta(path))
+            meta = _read_meta(path)
+            if not exists(path):
+                path = None
+            return File(path, digest, meta)
         elif isdir(path):
             return _lsdir(path, digest)
         elif exists(path):
@@ -226,6 +231,8 @@ class Blobs(object):
                 digest = join(rel_root, filename)
                 meta.append(('path', digest))
             if yield_files:
+                if not exists(path):
+                    path = None
                 yield File(path, digest, meta)
             else:
                 yield
@@ -275,7 +282,7 @@ def _write_meta(path, meta, seqno):
         for key, value in meta.items() if isinstance(meta, dict) else meta:
             if seqno is None and key == 'x-seqno':
                 seqno = int(value)
-            f.write('%s: %s\n' % (key, value))
+            f.write('%s: %s\n' % (key, toolkit.ascii(value)))
     os.utime(path, (seqno, seqno))
 
 
diff --git a/sugar_network/toolkit/packets.py b/sugar_network/toolkit/packets.py
index 46bc223..0960a1f 100644
--- a/sugar_network/toolkit/packets.py
+++ b/sugar_network/toolkit/packets.py
@@ -28,7 +28,6 @@ from os.path import dirname, exists, join
 
 from sugar_network import toolkit
 from sugar_network.toolkit.router import File
-from sugar_network.toolkit.coroutine import this
 from sugar_network.toolkit import http, coroutine, BUFFER_SIZE, enforce
 
 
@@ -60,17 +59,14 @@ def decode(stream, limit=None):
 
 
 def encode(items, limit=None, header=None, compresslevel=None,
-        on_complete=None, **kwargs):
+        on_complete=None, download_blobs=False, **kwargs):
     _logger.debug('Encode %r limit=%r header=%r', items, limit, header)
 
     if compresslevel is 0:
         ostream = _Encoder()
     else:
         ostream = _ZippedEncoder(compresslevel)
-
-    # In case of downloading blobs
-    # (?) reuse current `this.http`
-    this.http = http.Connection()
+    connection = http.Connection()
 
     if limit is None:
         limit = sys.maxint
@@ -116,6 +112,8 @@ def encode(items, limit=None, header=None, compresslevel=None,
                     if isinstance(record, File):
                         blob_len = record.size
                         chunk = record.meta
+                        if not record.path:
+                            chunk['digest'] = record.digest
                     else:
                         chunk = record
                     chunk = ostream.write_record(chunk,
@@ -130,8 +128,14 @@ def encode(items, limit=None, header=None, compresslevel=None,
                         continue
                     if chunk:
                         yield chunk
-                    if blob_len:
-                        for chunk in record.iter_content():
+                    if blob_len and (record.path or download_blobs):
+                        if record.path:
+                            blob_content = record.iter_content()
+                        else:
+                            url = record.meta['location']
+                            enforce(url, http.NotFound, 'No location')
+                            blob_content = connection.download(url)
+                        for chunk in blob_content:
                             blob_len -= len(chunk)
                             if not blob_len:
                                 chunk += '\n'
@@ -245,6 +249,10 @@ class _DecodeIterator(object):
             yield record
             return
 
+        if 'location' in record:
+            yield File(None, digest=record.pop('digest'), meta=record)
+            return
+
         blob_len = int(blob_len)
         with toolkit.NamedTemporaryFile() as blob:
             digest = hashlib.sha1()
diff --git a/sugar_network/toolkit/router.py b/sugar_network/toolkit/router.py
index bd5da32..ec89fec 100644
--- a/sugar_network/toolkit/router.py
+++ b/sugar_network/toolkit/router.py
@@ -413,21 +413,8 @@ class File(str):
             return basename(self.path)
 
     def iter_content(self):
-        if self.path:
-            return self._iter_content()
-        url = self.meta.get('location')
-        enforce(url, http.NotFound, 'No location')
-        blob = this.http.request('GET', url, allow_redirects=True,
-                # Request for uncompressed data
-                headers={'accept-encoding': ''})
-        self.meta.clear()
-        for tag in ('content-length', 'content-type', 'content-disposition'):
-            value = blob.headers.get(tag)
-            if value:
-                self.meta[tag] = value
-        return blob.iter_content(toolkit.BUFFER_SIZE)
-
-    def _iter_content(self):
+        if not self.path:
+            return
         with file(self.path, 'rb') as f:
             while True:
                 chunk = f.read(toolkit.BUFFER_SIZE)