From 0e931f9bbb075f66f951057adec96afbf8b1c792 Mon Sep 17 00:00:00 2001 From: Aleksey Lim Date: Fri, 15 Nov 2013 17:10:39 +0000 Subject: Default values for stats API command --- diff --git a/sugar_network/node/routes.py b/sugar_network/node/routes.py index 7b36b12..1182f28 100644 --- a/sugar_network/node/routes.py +++ b/sugar_network/node/routes.py @@ -35,7 +35,7 @@ from sugar_network.toolkit.bundle import Bundle from sugar_network.toolkit import pylru, http, coroutine, exception, enforce -_MAX_STAT_RECORDS = 128 +_MAX_STAT_RECORDS = 100 _AUTH_POOL_SIZE = 1024 _logger = logging.getLogger('node.routes') @@ -88,16 +88,16 @@ class NodeRoutes(model.VolumeRoutes, model.FrontRoutes): 'start': int, 'end': int, 'records': int, 'source': list}, mime_type='application/json') def stats(self, start, end, records, source): - if not source or records <= 0 or start > end: - return {} - enforce(self._stats is not None, 'Node stats is disabled') + if not source: + return {} if records > _MAX_STAT_RECORDS: _logger.debug('Decrease %d stats records number to %d', records, _MAX_STAT_RECORDS) records = _MAX_STAT_RECORDS - resolution = (end - start) / records + elif records <= 0: + records = _MAX_STAT_RECORDS / 10 stats = {} for i in source: @@ -105,7 +105,7 @@ class NodeRoutes(model.VolumeRoutes, model.FrontRoutes): db_name, ds_name = i.split('.', 1) stats.setdefault(db_name, []).append(ds_name) - return self._stats.report(stats, start, end, resolution) + return self._stats.report(stats, start, end, records) @route('POST', ['user'], mime_type='application/json') def register(self, request): diff --git a/sugar_network/node/stats_node.py b/sugar_network/node/stats_node.py index 0cba0b8..61a4a28 100644 --- a/sugar_network/node/stats_node.py +++ b/sugar_network/node/stats_node.py @@ -40,8 +40,6 @@ stats_node_rras = Option( ], type_cast=Option.list_cast, type_repr=Option.list_repr) -_MAX_FRAME = 64 - _logger = logging.getLogger('node.stats_node') @@ -71,17 +69,27 @@ class Sniffer(object): if values is not None: self._rrd[resource].put(values, timestamp=timestamp) - def report(self, dbs, start, end, resolution): + def report(self, dbs, start, end, records): result = {} - for rdb in self._rrd: - if rdb.name not in dbs: - continue + + rdbs = [self._rrd[i] for i in dbs if i in self._rrd] + if not rdbs: + return result + + if not start: + start = min([i.first for i in rdbs]) + if not end: + end = max([i.last for i in rdbs]) + resolution = max(1, (end - start) / records) + + for rdb in rdbs: info = result[rdb.name] = [] for ts, ds_values in rdb.get(start, end, resolution): values = {} for name in dbs[rdb.name]: values[name] = ds_values.get(name) info.append((ts, values)) + return result diff --git a/sugar_network/toolkit/rrd.py b/sugar_network/toolkit/rrd.py index 6b5d6e8..a5f879d 100644 --- a/sugar_network/toolkit/rrd.py +++ b/sugar_network/toolkit/rrd.py @@ -62,6 +62,9 @@ class Rrd(object): def __getitem__(self, name): return self.get(name) + def __contains__(self, name): + return name in self._dbsets + @property def root(self): return self._root @@ -199,8 +202,8 @@ class _DbSet(object): filename += '-%s' % revision filename += '.rrd' - _logger.debug('Create %s database in %s starting from %s', - filename, self._root, timestamp) + _logger.debug('Create %s database in %s start=%s step=%s', + filename, self._root, timestamp, self._step) fields = [] for name in field_names: diff --git a/tests/__init__.py b/tests/__init__.py index fdc7449..58203a9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -103,7 +103,8 @@ class Test(unittest.TestCase): mountpoints._found.clear() mountpoints._COMPLETE_MOUNT_TIMEOUT = .1 stats_node.stats_node.value = False - stats_node.stats_node_step.value = 0 + stats_node.stats_node_step.value = 1 + stats_node.stats_node_rras.value = ['RRA:AVERAGE:0.5:1:60'] stats_user.stats_user.value = False stats_user.stats_user_step.value = 1 stats_user._user_cache.clear() diff --git a/tests/units/node/node.py b/tests/units/node/node.py index 586fbd6..b5abe78 100755 --- a/tests/units/node/node.py +++ b/tests/units/node/node.py @@ -6,6 +6,7 @@ import os import time import json import base64 +import shutil import hashlib from M2Crypto import RSA from email.utils import formatdate, parsedate @@ -106,11 +107,11 @@ class NodeTest(tests.Test): def test_NodeStats(self): stats_node.stats_node.value = True - stats_node.stats_node_step.value = 1 + stats_node.stats_node_rras.value = ['RRA:AVERAGE:0.5:1:60', 'RRA:AVERAGE:0.5:3:60'] rrd = Rrd('stats/node', stats_node.stats_node_step.value, stats_node.stats_node_rras.value) ts = int(time.time()) / 3 * 3 - for i in range(100): + for i in range(10): rrd['user'].put({'total': i}, ts + i) volume = db.Volume('db', model.RESOURCES) @@ -131,10 +132,36 @@ class NodeTest(tests.Test): (ts + 3, {'total': 2.0}), (ts + 6, {'total': 5.0}), (ts + 9, {'total': 8.0}), - (ts + 12, {'total': 11.0}), ], }, - call(cp, method='GET', cmd='stats', source='user.total', start=ts, end=ts + 12, records=4)) + call(cp, method='GET', cmd='stats', source='user.total', start=ts, end=ts + 9, records=3)) + + def test_NodeStatsDefaults(self): + stats_node.stats_node.value = True + rrd = Rrd('stats/node', stats_node.stats_node_step.value, stats_node.stats_node_rras.value) + + ts = int(time.time()) + for i in range(10): + rrd['user'].put({'total': i}, ts + i) + + volume = db.Volume('db', model.RESOURCES) + cp = NodeRoutes('guid', volume) + + self.assertEqual({ + 'user': [ + (ts + 0, {'total': 0.0}), + (ts + 1, {'total': 1.0}), + (ts + 2, {'total': 2.0}), + (ts + 3, {'total': 3.0}), + (ts + 4, {'total': 4.0}), + (ts + 5, {'total': 5.0}), + (ts + 6, {'total': 6.0}), + (ts + 7, {'total': 7.0}), + (ts + 8, {'total': 8.0}), + (ts + 9, {'total': 9.0}), + ], + }, + call(cp, method='GET', cmd='stats', source='user.total')) def test_HandleDeletes(self): volume = db.Volume('db', model.RESOURCES) diff --git a/tests/units/node/stats_node.py b/tests/units/node/stats_node.py index 60e7d96..a0ae5d0 100755 --- a/tests/units/node/stats_node.py +++ b/tests/units/node/stats_node.py @@ -6,7 +6,7 @@ import time from __init__ import tests from sugar_network import db, model -from sugar_network.node.stats_node import stats_node_step, Sniffer +from sugar_network.node.stats_node import Sniffer from sugar_network.toolkit.rrd import Rrd from sugar_network.toolkit.router import Request @@ -194,7 +194,6 @@ class StatsTest(tests.Test): self.assertEqual(1, stats._stats['artifact'].downloaded) def test_Commit(self): - stats_node_step.value = 1 volume = db.Volume('local', model.RESOURCES) volume['user'].create({'guid': 'user', 'name': 'user', 'color': '', 'pubkey': ''}) volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) @@ -273,7 +272,6 @@ class StatsTest(tests.Test): [[(j.name,) + i for i in j.get(j.last, j.last)] for j in Rrd('stats/node', 1)]) def test_CommitContextStats(self): - stats_node_step.value = 1 volume = db.Volume('local', model.RESOURCES) volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) @@ -323,7 +321,6 @@ class StatsTest(tests.Test): self.assertEqual(3, volume['context'].get('context')['rating']) def test_CommitArtifactStats(self): - stats_node_step.value = 1 volume = db.Volume('local', model.RESOURCES) volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) diff --git a/tests/units/node/volume.py b/tests/units/node/volume.py index 8bd10df..40e052b 100755 --- a/tests/units/node/volume.py +++ b/tests/units/node/volume.py @@ -11,7 +11,7 @@ from __init__ import tests from sugar_network import db, toolkit, model from sugar_network.node.volume import diff, merge -from sugar_network.node.stats_node import stats_node_step, Sniffer +from sugar_network.node.stats_node import Sniffer from sugar_network.node.routes import NodeRoutes from sugar_network.toolkit.rrd import Rrd from sugar_network.toolkit.router import Router, Request, Response, fallbackroute, Blob, ACL, route @@ -349,7 +349,6 @@ class VolumeTest(tests.Test): assert volume['document'].exists('1') def test_merge_UpdateStats(self): - stats_node_step.value = 1 volume = db.Volume('db', model.RESOURCES) cp = NodeRoutes('guid', volume) stats = Sniffer(volume) -- cgit v0.9.1