Do not go offline on gateway timeouts

5 files changed, 62 insertions, 19 deletions

diff --git a/sugar_network/client/routes.py b/sugar_network/client/routes.py
index c344ab1..e9df9e1 100644
--- a/sugar_network/client/routes.py
+++ b/sugar_network/client/routes.py
@@ -267,7 +267,7 @@ class ClientRoutes(model.FrontRoutes, implementations.Routes, journal.Routes):
             _logger.debug('Connecting to %r node', url)
             self._node = client.Connection(url)
             info = self._node.get(cmd='info')
-            impl_info = info['documents'].get('implementation')
+            impl_info = info['resources'].get('implementation')
             if impl_info:
                 self.invalidate_solutions(impl_info['mtime'])
             if self._inline.is_set():
@@ -287,13 +287,11 @@ class ClientRoutes(model.FrontRoutes, implementations.Routes, journal.Routes):
                             if self._no_subscription:
                                 return
                             pull_events()
-                        except http.HTTPError, error:
-                            if error.response.status_code in (502, 504):
-                                _logger.debug('Retry %r on gateway error', url)
-                                continue
+                        except (http.BadGateway, http.GatewayTimeout):
+                            _logger.debug('Retry %r on gateway error', url)
+                            continue
                         except Exception:
-                            toolkit.exception(_logger,
-                                    'Connection to %r failed', url)
+                            _logger.exception('Connection to %r failed', url)
                         break
                 self._got_offline()
                 if not timeout:
@@ -363,8 +361,7 @@ class CachedClientRoutes(ClientRoutes):
             try:
                 self._node.call(request)
             except Exception:
-                toolkit.exception(_logger,
-                        'Cannot push %r, will postpone', request)
+                _logger.exception('Cannot push %r, will postpone', request)
                 skiped_seq.include(seq)
             else:
                 pushed_seq.include(seq)
@@ -494,8 +491,7 @@ class _NodeRoutes(SlaveRoutes, Router):
                         join(mountpoint, _SYNC_DIRNAME),
                         **(self._offline_session or {}))
             except Exception, error:
-                toolkit.exception(_logger,
-                        'Failed to complete synchronization')
+                _logger.exception('Failed to complete synchronization')
                 self._localcast({'event': 'sync_abort', 'error': str(error)})
                 self._offline_session = None
                 raise
diff --git a/sugar_network/node/routes.py b/sugar_network/node/routes.py
index 4b8b993..b117e98 100644
--- a/sugar_network/node/routes.py
+++ b/sugar_network/node/routes.py
@@ -71,7 +71,7 @@ class NodeRoutes(model.VolumeRoutes, model.FrontRoutes):
         documents = {}
         for name, directory in self.volume.items():
             documents[name] = {'mtime': directory.mtime}
-        return {'guid': self._guid, 'documents': documents}
+        return {'guid': self._guid, 'resources': documents}
 
     @route('GET', cmd='stats', arguments={
                 'start': int, 'end': int, 'resolution': int, 'source': list},
diff --git a/sugar_network/toolkit/http.py b/sugar_network/toolkit/http.py
index d57d3ce..cadf64c 100644
--- a/sugar_network/toolkit/http.py
+++ b/sugar_network/toolkit/http.py
@@ -85,12 +85,24 @@ class NotFound(Status):
     status_code = 404
 
 
+class BadGateway(Status):
+
+    status = '502 Bad Gateway'
+    status_code = 502
+
+
 class ServiceUnavailable(Status):
 
     status = '503 Service Unavailable'
     status_code = 503
 
 
+class GatewayTimeout(Status):
+
+    status = '504 Gateway Timeout'
+    status_code = 504
+
+
 def download(url, dst_path=None):
     # TODO (?) Reuse HTTP session
     return Connection().download(url, dst_path)
@@ -233,12 +245,8 @@ class Connection(object):
                         continue
                     error = content or reply.headers.get('x-sn-error') or \
                             'No error message provided'
-                _logger.debug('Request failed, method=%s path=%r params=%r '
-                        'headers=%r status_code=%s error=%s',
-                        method, path, params, headers, reply.status_code,
-                        '\n' + error)
                 cls = _FORWARD_STATUSES.get(reply.status_code, RuntimeError)
-                raise cls(error)
+                raise cls, error, sys.exc_info()[2]
             break
 
         return reply
@@ -385,5 +393,7 @@ _FORWARD_STATUSES = {
         BadRequest.status_code: BadRequest,
         Forbidden.status_code: Forbidden,
         NotFound.status_code: NotFound,
+        BadGateway.status_code: BadGateway,
         ServiceUnavailable.status_code: ServiceUnavailable,
+        GatewayTimeout.status_code: GatewayTimeout,
         }
diff --git a/tests/__init__.py b/tests/__init__.py
index adbd83f..ababed5 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -262,11 +262,11 @@ class Test(unittest.TestCase):
     def create_mountset(self, classes=None):
         self.start_server(classes, root=False)
 
-    def start_master(self, classes=None):
+    def start_master(self, classes=None, routes=MasterRoutes):
         if classes is None:
             classes = [User, Context, Implementation]
         self.node_volume = db.Volume('master', classes)
-        cp = MasterRoutes('guid', self.node_volume)
+        cp = routes('guid', self.node_volume)
         r = Router(cp)
         self.node = coroutine.WSGIServer(('127.0.0.1', 8888), Router(cp))
         coroutine.spawn(self.node.serve_forever)
diff --git a/tests/units/client/online_routes.py b/tests/units/client/online_routes.py
index 391e676..786b07f 100755
--- a/tests/units/client/online_routes.py
+++ b/tests/units/client/online_routes.py
@@ -1327,6 +1327,43 @@ Can't find all required implementations:
         self.fork_master([User])
         self.wait_for_events(ipc, event='inline', state='online').wait()
 
+    def test_SilentReconnectOnGatewayErrors(self):
+
+        class Routes(object):
+
+            subscribe_tries = 0
+
+            def __init__(self, *args):
+                pass
+
+            @route('GET', cmd='info', mime_type='application/json')
+            def info(self):
+                return {'resources': {}}
+
+            @route('GET', cmd='subscribe', mime_type='text/event-stream')
+            def subscribe(self, request=None, response=None, ping=False, **condition):
+                Routes.subscribe_tries += 1
+                coroutine.sleep(.1)
+                if Routes.subscribe_tries % 2:
+                    raise http.BadGateway()
+                else:
+                    raise http.GatewayTimeout()
+
+        node_pid = self.start_master([User], Routes)
+        self.start_client([User])
+        ipc = IPCConnection()
+        self.wait_for_events(ipc, event='inline', state='online').wait()
+
+        def read_events():
+            for event in ipc.subscribe():
+                events.append(event)
+        events = []
+        coroutine.spawn(read_events)
+
+        coroutine.sleep(1)
+        self.assertEqual([], events)
+        assert Routes.subscribe_tries > 2
+
     def test_inline(self):
         cp = ClientRoutes(Volume('client', model.RESOURCES), client.api_url.value)
         assert not cp.inline()