From a181951a95ea19225807aa7c716347192c8af1ee Mon Sep 17 00:00:00 2001 From: Aleksey Lim Date: Sat, 23 Nov 2013 07:11:37 +0000 Subject: Replace rrd ABSOLUTE fields by GAUGE to easy understand node stats --- diff --git a/sugar-network b/sugar-network index 6f83100..8957f99 100755 --- a/sugar-network +++ b/sugar-network @@ -289,12 +289,13 @@ class Application(application.Application): else: if type(value) not in (list, tuple): value = [value] + term = '\n' if len(value) > 5 else '\t' for n, i in enumerate(value): if n: - self._print('\t') + self._print(term) if type(i) is dict and len(i) == 1: i = i.values()[0] - self._print(i) + self._print('%s' % i) self._print('\n') if type(result) in (list, tuple): diff --git a/sugar_network/node/stats_node.py b/sugar_network/node/stats_node.py index 53f4676..fbb22fa 100644 --- a/sugar_network/node/stats_node.py +++ b/sugar_network/node/stats_node.py @@ -42,6 +42,8 @@ stats_node_rras = Option( ], type_cast=Option.list_cast, type_repr=Option.list_repr) +_HEARTBEAT = 60 * 60 * 24 * 365 + _logger = logging.getLogger('node.stats_node') @@ -57,17 +59,14 @@ class Sniffer(object): self._last = int(time.time()) for name, cls in _STATS.items(): - stats = self._stats[name] = cls(self._stats, volume, reset) + stats = self._stats[name] = cls(self._stats, volume) fields = {} for field in stats: - if field == 'total': - dst = 'GAUGE' - limit = 60 * 60 * 24 * 365 - else: - dst = 'ABSOLUTE' - limit = stats_node_step.value - fields[field] = 'DS:%s:%s:%s:U:U' % (field, dst, limit) + fields[field] = 'DS:%s:GAUGE:%s:U:U' % (field, _HEARTBEAT) if fields: + if not reset: + stats.update(self._rrd[name].last_ds) + stats['total'] = volume[name].find(limit=0)[1] self._rrd[name].fields = fields if exists(self._suspend_path): @@ -106,9 +105,6 @@ class Sniffer(object): if resource not in self._rrd: continue values = stats.copy() - for field in stats: - if field != 'total': - stats[field] = 0 if extra_values and resource in extra_values: values.update(extra_values[resource]) if values: @@ -176,7 +172,7 @@ class _Stats(dict): RESOURCE = None OWNERS = [] - def __init__(self, stats, volume, reset): + def __init__(self, stats, volume): self.objects = {} self._stats = stats self._volume = volume @@ -194,12 +190,9 @@ class _Stats(dict): class _ResourceStats(_Stats): - def __init__(self, stats, volume, reset): - _Stats.__init__(self, stats, volume, reset) - if reset: - self['total'] = 0 - else: - self['total'] = volume[self.RESOURCE].find(limit=0)[1] + def __init__(self, stats, volume): + _Stats.__init__(self, stats, volume) + self['total'] = 0 def log(self, request): if request.method == 'POST': @@ -250,8 +243,8 @@ class _ContextStats(_ResourceStats): RESOURCE = 'context' - def __init__(self, stats, volume, reset): - _ResourceStats.__init__(self, stats, volume, reset) + def __init__(self, stats, volume): + _ResourceStats.__init__(self, stats, volume) self['released'] = 0 self['failed'] = 0 self['downloaded'] = 0 @@ -318,8 +311,8 @@ class _ArtifactStats(_ResourceStats): RESOURCE = 'artifact' OWNERS = ['context'] - def __init__(self, stats, volume, reset): - _ResourceStats.__init__(self, stats, volume, reset) + def __init__(self, stats, volume): + _ResourceStats.__init__(self, stats, volume) self['downloaded'] = 0 def log(self, request): diff --git a/sugar_network/toolkit/rrd.py b/sugar_network/toolkit/rrd.py index da58aff..bafe6d1 100644 --- a/sugar_network/toolkit/rrd.py +++ b/sugar_network/toolkit/rrd.py @@ -107,13 +107,25 @@ class _DbSet(object): @property def first(self): - if self._revisions: - return self._revisions[0].first + if not self._revisions: + return + return self._revisions[0].first @property def last(self): - if self._revisions: - return self._revisions[-1].last + if not self._revisions: + return + return self._revisions[-1].last + + @property + def last_ds(self): + if not self._revisions or not self._field_names: + return {} + info = _rrdtool.info(self._revisions[-1].path) + result = {} + for field in self._field_names: + result[field] = float(info.get('ds[%s].last_ds' % field) or 0) + return result def load(self, filename, revision): _logger.debug('Load %s database from %s with revision %s', diff --git a/tests/units/node/node.py b/tests/units/node/node.py index 9d6ae6f..f7d4fa7 100755 --- a/tests/units/node/node.py +++ b/tests/units/node/node.py @@ -1231,14 +1231,14 @@ class NodeTest(tests.Test): ], 'context': [ (ts + 1, {'total': 1.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 1.0}), - (ts + 2, {'total': 1.0, 'released': 1.0, 'failed': 0.0, 'downloaded': 0.0}), - (ts + 3, {'total': 1.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}), - (ts + 4, {'total': 2.0, 'released': 2.0, 'failed': 2.0, 'downloaded': 2.0}), - (ts + 5, {'total': 2.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}), - (ts + 6, {'total': 2.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}), - (ts + 7, {'total': 2.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}), - (ts + 8, {'total': 2.0, 'released': 0.0, 'failed': 1.0, 'downloaded': 0.0}), - (ts + 9, {'total': 2.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}), + (ts + 2, {'total': 1.0, 'released': 1.0, 'failed': 0.0, 'downloaded': 1.0}), + (ts + 3, {'total': 1.0, 'released': 1.0, 'failed': 0.0, 'downloaded': 1.0}), + (ts + 4, {'total': 2.0, 'released': 3.0, 'failed': 2.0, 'downloaded': 3.0}), + (ts + 5, {'total': 2.0, 'released': 3.0, 'failed': 2.0, 'downloaded': 3.0}), + (ts + 6, {'total': 2.0, 'released': 3.0, 'failed': 2.0, 'downloaded': 3.0}), + (ts + 7, {'total': 2.0, 'released': 3.0, 'failed': 2.0, 'downloaded': 3.0}), + (ts + 8, {'total': 2.0, 'released': 3.0, 'failed': 3.0, 'downloaded': 3.0}), + (ts + 9, {'total': 2.0, 'released': 3.0, 'failed': 3.0, 'downloaded': 3.0}), ], 'review': [ (ts + 1, {'total': 0.0}), @@ -1275,14 +1275,14 @@ class NodeTest(tests.Test): ], 'artifact': [ (ts + 1, {'total': 0.0, 'downloaded': 1.0}), - (ts + 2, {'total': 0.0, 'downloaded': 0.0}), - (ts + 3, {'total': 1.0, 'downloaded': 0.0}), - (ts + 4, {'total': 2.0, 'downloaded': 0.0}), - (ts + 5, {'total': 2.0, 'downloaded': 0.0}), - (ts + 6, {'total': 2.0, 'downloaded': 0.0}), - (ts + 7, {'total': 2.0, 'downloaded': 2.0}), - (ts + 8, {'total': 2.0, 'downloaded': 0.0}), - (ts + 9, {'total': 2.0, 'downloaded': 0.0}), + (ts + 2, {'total': 0.0, 'downloaded': 1.0}), + (ts + 3, {'total': 1.0, 'downloaded': 1.0}), + (ts + 4, {'total': 2.0, 'downloaded': 1.0}), + (ts + 5, {'total': 2.0, 'downloaded': 1.0}), + (ts + 6, {'total': 2.0, 'downloaded': 1.0}), + (ts + 7, {'total': 2.0, 'downloaded': 3.0}), + (ts + 8, {'total': 2.0, 'downloaded': 3.0}), + (ts + 9, {'total': 2.0, 'downloaded': 3.0}), ], 'comment': [ (ts + 1, {'total': 0.0}), diff --git a/tests/units/node/stats_node.py b/tests/units/node/stats_node.py index aec9102..123d225 100755 --- a/tests/units/node/stats_node.py +++ b/tests/units/node/stats_node.py @@ -378,7 +378,7 @@ class StatsTest(tests.Test): rdb = Rrd('stats', 1)['context'] self.assertEqual([ (ts + 0, {'failed': 0.0, 'downloaded': 0.0, 'total': 0.0, 'released': 0.0}), - (ts + 5, {'failed': 0.0, 'downloaded': 0.2, 'total': 2.0, 'released': 0.0}), + (ts + 5, {'failed': 0.0, 'downloaded': 1.0, 'total': 2.0, 'released': 0.0}), ], [i for i in rdb.get(ts, ts + 20)]) @@ -396,7 +396,7 @@ class StatsTest(tests.Test): rdb = Rrd('stats', 1)['context'] self.assertEqual([ (ts + 0, {'failed': 0.0, 'downloaded': 0.0, 'total': 0.0, 'released': 0.0}), - (ts + 5, {'failed': 0.0, 'downloaded': 0.2, 'total': 2.0, 'released': 0.0}), + (ts + 5, {'failed': 0.0, 'downloaded': 1.0, 'total': 2.0, 'released': 0.0}), ], [i for i in rdb.get(ts, ts + 10)]) @@ -406,8 +406,8 @@ class StatsTest(tests.Test): rdb = Rrd('stats', 1)['context'] self.assertEqual([ (ts + 0, {'failed': 0.0, 'downloaded': 0.0, 'total': 0.0, 'released': 0.0}), - (ts + 5, {'failed': 0.0, 'downloaded': 0.2, 'total': 2.0, 'released': 0.0}), - (ts + 10, {'failed': 0.0, 'downloaded': 0.4, 'total': 3.0, 'released': 0.0}), + (ts + 5, {'failed': 0.0, 'downloaded': 1.0, 'total': 2.0, 'released': 0.0}), + (ts + 10, {'failed': 0.0, 'downloaded': 3.0, 'total': 3.0, 'released': 0.0}), ], [i for i in rdb.get(ts, ts + 20)]) diff --git a/tests/units/node/volume.py b/tests/units/node/volume.py index 262b85d..7978ef0 100755 --- a/tests/units/node/volume.py +++ b/tests/units/node/volume.py @@ -527,7 +527,7 @@ class VolumeTest(tests.Test): 'failed': 0.0, 'downloaded': 0.0, 'total': 1.0, - 'released': 0.0, + 'released': 1.0, })], ], [[(j.name,) + i for i in j.get(j.last, j.last)] for j in Rrd('stats/node', 1)]) @@ -579,7 +579,7 @@ class VolumeTest(tests.Test): 'failed': 0.0, 'downloaded': 0.0, 'total': 0.0, - 'released': 0.0, + 'released': 1.0, })], ], [[(j.name,) + i for i in j.get(j.last, j.last)] for j in Rrd('stats/node', 1)]) -- cgit v0.9.1