Replace rrd ABSOLUTE fields by GAUGE to easy understand node stats

6 files changed, 56 insertions, 50 deletions

diff --git a/sugar-network b/sugar-network
index 6f83100..8957f99 100755
--- a/sugar-network
+++ b/sugar-network
@@ -289,12 +289,13 @@ class Application(application.Application):
             else:
                 if type(value) not in (list, tuple):
                     value = [value]
+                term = '\n' if len(value) > 5 else '\t'
                 for n, i in enumerate(value):
                     if n:
-                        self._print('\t')
+                        self._print(term)
                     if type(i) is dict and len(i) == 1:
                         i = i.values()[0]
-                    self._print(i)
+                    self._print('%s' % i)
                 self._print('\n')
 
         if type(result) in (list, tuple):
diff --git a/sugar_network/node/stats_node.py b/sugar_network/node/stats_node.py
index 53f4676..fbb22fa 100644
--- a/sugar_network/node/stats_node.py
+++ b/sugar_network/node/stats_node.py
@@ -42,6 +42,8 @@ stats_node_rras = Option(
             ],
         type_cast=Option.list_cast, type_repr=Option.list_repr)
 
+_HEARTBEAT = 60 * 60 * 24 * 365
+
 _logger = logging.getLogger('node.stats_node')
 
 
@@ -57,17 +59,14 @@ class Sniffer(object):
         self._last = int(time.time())
 
         for name, cls in _STATS.items():
-            stats = self._stats[name] = cls(self._stats, volume, reset)
+            stats = self._stats[name] = cls(self._stats, volume)
             fields = {}
             for field in stats:
-                if field == 'total':
-                    dst = 'GAUGE'
-                    limit = 60 * 60 * 24 * 365
-                else:
-                    dst = 'ABSOLUTE'
-                    limit = stats_node_step.value
-                fields[field] = 'DS:%s:%s:%s:U:U' % (field, dst, limit)
+                fields[field] = 'DS:%s:GAUGE:%s:U:U' % (field, _HEARTBEAT)
             if fields:
+                if not reset:
+                    stats.update(self._rrd[name].last_ds)
+                    stats['total'] = volume[name].find(limit=0)[1]
                 self._rrd[name].fields = fields
 
         if exists(self._suspend_path):
@@ -106,9 +105,6 @@ class Sniffer(object):
             if resource not in self._rrd:
                 continue
             values = stats.copy()
-            for field in stats:
-                if field != 'total':
-                    stats[field] = 0
             if extra_values and resource in extra_values:
                 values.update(extra_values[resource])
             if values:
@@ -176,7 +172,7 @@ class _Stats(dict):
     RESOURCE = None
     OWNERS = []
 
-    def __init__(self, stats, volume, reset):
+    def __init__(self, stats, volume):
         self.objects = {}
         self._stats = stats
         self._volume = volume
@@ -194,12 +190,9 @@ class _Stats(dict):
 
 class _ResourceStats(_Stats):
 
-    def __init__(self, stats, volume, reset):
-        _Stats.__init__(self, stats, volume, reset)
-        if reset:
-            self['total'] = 0
-        else:
-            self['total'] = volume[self.RESOURCE].find(limit=0)[1]
+    def __init__(self, stats, volume):
+        _Stats.__init__(self, stats, volume)
+        self['total'] = 0
 
     def log(self, request):
         if request.method == 'POST':
@@ -250,8 +243,8 @@ class _ContextStats(_ResourceStats):
 
     RESOURCE = 'context'
 
-    def __init__(self, stats, volume, reset):
-        _ResourceStats.__init__(self, stats, volume, reset)
+    def __init__(self, stats, volume):
+        _ResourceStats.__init__(self, stats, volume)
         self['released'] = 0
         self['failed'] = 0
         self['downloaded'] = 0
@@ -318,8 +311,8 @@ class _ArtifactStats(_ResourceStats):
     RESOURCE = 'artifact'
     OWNERS = ['context']
 
-    def __init__(self, stats, volume, reset):
-        _ResourceStats.__init__(self, stats, volume, reset)
+    def __init__(self, stats, volume):
+        _ResourceStats.__init__(self, stats, volume)
         self['downloaded'] = 0
 
     def log(self, request):
diff --git a/sugar_network/toolkit/rrd.py b/sugar_network/toolkit/rrd.py
index da58aff..bafe6d1 100644
--- a/sugar_network/toolkit/rrd.py
+++ b/sugar_network/toolkit/rrd.py
@@ -107,13 +107,25 @@ class _DbSet(object):
 
     @property
     def first(self):
-        if self._revisions:
-            return self._revisions[0].first
+        if not self._revisions:
+            return
+        return self._revisions[0].first
 
     @property
     def last(self):
-        if self._revisions:
-            return self._revisions[-1].last
+        if not self._revisions:
+            return
+        return self._revisions[-1].last
+
+    @property
+    def last_ds(self):
+        if not self._revisions or not self._field_names:
+            return {}
+        info = _rrdtool.info(self._revisions[-1].path)
+        result = {}
+        for field in self._field_names:
+            result[field] = float(info.get('ds[%s].last_ds' % field) or 0)
+        return result
 
     def load(self, filename, revision):
         _logger.debug('Load %s database from %s with revision %s',
diff --git a/tests/units/node/node.py b/tests/units/node/node.py
index 9d6ae6f..f7d4fa7 100755
--- a/tests/units/node/node.py
+++ b/tests/units/node/node.py
@@ -1231,14 +1231,14 @@ class NodeTest(tests.Test):
                 ],
             'context': [
                 (ts + 1, {'total': 1.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 1.0}),
-                (ts + 2, {'total': 1.0, 'released': 1.0, 'failed': 0.0, 'downloaded': 0.0}),
-                (ts + 3, {'total': 1.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}),
-                (ts + 4, {'total': 2.0, 'released': 2.0, 'failed': 2.0, 'downloaded': 2.0}),
-                (ts + 5, {'total': 2.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}),
-                (ts + 6, {'total': 2.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}),
-                (ts + 7, {'total': 2.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}),
-                (ts + 8, {'total': 2.0, 'released': 0.0, 'failed': 1.0, 'downloaded': 0.0}),
-                (ts + 9, {'total': 2.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}),
+                (ts + 2, {'total': 1.0, 'released': 1.0, 'failed': 0.0, 'downloaded': 1.0}),
+                (ts + 3, {'total': 1.0, 'released': 1.0, 'failed': 0.0, 'downloaded': 1.0}),
+                (ts + 4, {'total': 2.0, 'released': 3.0, 'failed': 2.0, 'downloaded': 3.0}),
+                (ts + 5, {'total': 2.0, 'released': 3.0, 'failed': 2.0, 'downloaded': 3.0}),
+                (ts + 6, {'total': 2.0, 'released': 3.0, 'failed': 2.0, 'downloaded': 3.0}),
+                (ts + 7, {'total': 2.0, 'released': 3.0, 'failed': 2.0, 'downloaded': 3.0}),
+                (ts + 8, {'total': 2.0, 'released': 3.0, 'failed': 3.0, 'downloaded': 3.0}),
+                (ts + 9, {'total': 2.0, 'released': 3.0, 'failed': 3.0, 'downloaded': 3.0}),
                 ],
             'review': [
                 (ts + 1, {'total': 0.0}),
@@ -1275,14 +1275,14 @@ class NodeTest(tests.Test):
                 ],
             'artifact': [
                 (ts + 1, {'total': 0.0, 'downloaded': 1.0}),
-                (ts + 2, {'total': 0.0, 'downloaded': 0.0}),
-                (ts + 3, {'total': 1.0, 'downloaded': 0.0}),
-                (ts + 4, {'total': 2.0, 'downloaded': 0.0}),
-                (ts + 5, {'total': 2.0, 'downloaded': 0.0}),
-                (ts + 6, {'total': 2.0, 'downloaded': 0.0}),
-                (ts + 7, {'total': 2.0, 'downloaded': 2.0}),
-                (ts + 8, {'total': 2.0, 'downloaded': 0.0}),
-                (ts + 9, {'total': 2.0, 'downloaded': 0.0}),
+                (ts + 2, {'total': 0.0, 'downloaded': 1.0}),
+                (ts + 3, {'total': 1.0, 'downloaded': 1.0}),
+                (ts + 4, {'total': 2.0, 'downloaded': 1.0}),
+                (ts + 5, {'total': 2.0, 'downloaded': 1.0}),
+                (ts + 6, {'total': 2.0, 'downloaded': 1.0}),
+                (ts + 7, {'total': 2.0, 'downloaded': 3.0}),
+                (ts + 8, {'total': 2.0, 'downloaded': 3.0}),
+                (ts + 9, {'total': 2.0, 'downloaded': 3.0}),
                 ],
             'comment': [
                 (ts + 1, {'total': 0.0}),
diff --git a/tests/units/node/stats_node.py b/tests/units/node/stats_node.py
index aec9102..123d225 100755
--- a/tests/units/node/stats_node.py
+++ b/tests/units/node/stats_node.py
@@ -378,7 +378,7 @@ class StatsTest(tests.Test):
         rdb = Rrd('stats', 1)['context']
         self.assertEqual([
             (ts + 0, {'failed': 0.0, 'downloaded': 0.0, 'total': 0.0, 'released': 0.0}),
-            (ts + 5, {'failed': 0.0, 'downloaded': 0.2, 'total': 2.0, 'released': 0.0}),
+            (ts + 5, {'failed': 0.0, 'downloaded': 1.0, 'total': 2.0, 'released': 0.0}),
             ],
             [i for i in rdb.get(ts, ts + 20)])
 
@@ -396,7 +396,7 @@ class StatsTest(tests.Test):
         rdb = Rrd('stats', 1)['context']
         self.assertEqual([
             (ts + 0, {'failed': 0.0, 'downloaded': 0.0, 'total': 0.0, 'released': 0.0}),
-            (ts + 5, {'failed': 0.0, 'downloaded': 0.2, 'total': 2.0, 'released': 0.0}),
+            (ts + 5, {'failed': 0.0, 'downloaded': 1.0, 'total': 2.0, 'released': 0.0}),
             ],
             [i for i in rdb.get(ts, ts + 10)])
 
@@ -406,8 +406,8 @@ class StatsTest(tests.Test):
         rdb = Rrd('stats', 1)['context']
         self.assertEqual([
             (ts + 0, {'failed': 0.0, 'downloaded': 0.0, 'total': 0.0, 'released': 0.0}),
-            (ts + 5, {'failed': 0.0, 'downloaded': 0.2, 'total': 2.0, 'released': 0.0}),
-            (ts + 10, {'failed': 0.0, 'downloaded': 0.4, 'total': 3.0, 'released': 0.0}),
+            (ts + 5, {'failed': 0.0, 'downloaded': 1.0, 'total': 2.0, 'released': 0.0}),
+            (ts + 10, {'failed': 0.0, 'downloaded': 3.0, 'total': 3.0, 'released': 0.0}),
             ],
             [i for i in rdb.get(ts, ts + 20)])
 
diff --git a/tests/units/node/volume.py b/tests/units/node/volume.py
index 262b85d..7978ef0 100755
--- a/tests/units/node/volume.py
+++ b/tests/units/node/volume.py
@@ -527,7 +527,7 @@ class VolumeTest(tests.Test):
                 'failed': 0.0,
                 'downloaded': 0.0,
                 'total': 1.0,
-                'released': 0.0,
+                'released': 1.0,
                 })],
             ],
             [[(j.name,) + i for i in j.get(j.last, j.last)] for j in Rrd('stats/node', 1)])
@@ -579,7 +579,7 @@ class VolumeTest(tests.Test):
                 'failed': 0.0,
                 'downloaded': 0.0,
                 'total': 0.0,
-                'released': 0.0,
+                'released': 1.0,
                 })],
             ],
             [[(j.name,) + i for i in j.get(j.last, j.last)] for j in Rrd('stats/node', 1)])