From ebb59c1da33fddff54529860ad3b12aaa41ea4e8 Mon Sep 17 00:00:00 2001 From: Aleksey Lim Date: Thu, 21 Nov 2013 20:26:30 +0000 Subject: Regenerate node stats --- diff --git a/doc/objects.dia b/doc/objects.dia index 5ff9450..2c23de0 100644 --- a/doc/objects.dia +++ b/doc/objects.dia @@ -455,7 +455,7 @@ - + @@ -464,7 +464,7 @@ - + #User# @@ -556,29 +556,6 @@ - #pubkey# - - - #str [WN]# - - - ## - - - #DSA public key for the key generated by Sugar Shell in user's profile# - - - - - - - - - - - - - #name# @@ -602,62 +579,16 @@ - #color# - - - #str [R]# - - - ## - - - #Sugar colors pair in format "#RRGGBB,#RRGGBB"# - - - - - - - - - - - - - - #machine_sn# - - - #str [WA]# - - - #""# - - - #XO serial number# - - - - - - - - - - - - - - #machine_uuid# + #location# - #str [WA]# + #str [R WA F]# #""# - #XO UUID# + #User's real address in arbitrary form# @@ -671,16 +602,16 @@ - #location# + #birthday# - #str [R WA F]# + #int [R WA S]# - #""# + #0# - #User's real address in arbitrary form# + # User's birthday in seconds from UNIX epoch# @@ -694,16 +625,16 @@ - #birthday# + #pubkey# - #int [R WA S]# + #str [WN]# - #0# + ## - #User's birthday in seconds from UNIX epoch# + #DSA public key for the key generated by Sugar Shell in user's profile# diff --git a/sugar-network-node b/sugar-network-node index cd6961a..07bd96d 100755 --- a/sugar-network-node +++ b/sugar-network-node @@ -25,6 +25,7 @@ from gevent import monkey import sugar_network_webui as webui from sugar_network import db, node, client, toolkit from sugar_network.node import stats_node, stats_user, obs +from sugar_network.node.routes import generate_node_stats from sugar_network.node.master import MasterRoutes from sugar_network.node.slave import SlaveRoutes from sugar_network.model import RESOURCES @@ -118,6 +119,14 @@ class Application(application.Daemon): path = self.args.pop(0) self._ensure_instance().post(cmd='offline-sync', path=path) + @application.command( + 're-generate node statistics', name='restat') + def restat(self): + enforce(not self.check_for_instance(), 'Shutdown the server at first') + volume = db.Volume(node.data_root.value, RESOURCES) + volume.populate() + generate_node_stats(volume, join(node.stats_root.value, 'node')) + def _ensure_instance(self): enforce(self.check_for_instance(), 'Node is not started') return Connection('http://localhost:%s' % diff --git a/sugar_network/client/routes.py b/sugar_network/client/routes.py index b4680c0..e43eea5 100644 --- a/sugar_network/client/routes.py +++ b/sugar_network/client/routes.py @@ -552,7 +552,6 @@ class _Auth(http.SugarAuth): import gconf conf = gconf.client_get_default() self._profile['name'] = conf.get_string('/desktop/sugar/user/nick') - self._profile['color'] = conf.get_string('/desktop/sugar/user/color') return http.SugarAuth.profile(self) def __call__(self, nonce): diff --git a/sugar_network/db/index.py b/sugar_network/db/index.py index 4de1382..ab40527 100644 --- a/sugar_network/db/index.py +++ b/sugar_network/db/index.py @@ -170,7 +170,7 @@ class IndexReader(object): enquire = self._enquire(request, query, order_by, group_by) mset = self._call_db(enquire.get_mset, offset, limit, check_at_least) - _logger.debug('Found in %s: %s time=%s total=%s parsed=%s', + _logger.debug('Found in %s: query=%r time=%s total=%s parsed=%s', self.metadata.name, query, time.time() - start_timestamp, mset.get_matches_estimated(), enquire.get_query()) diff --git a/sugar_network/model/user.py b/sugar_network/model/user.py index e35cce8..69d0d42 100644 --- a/sugar_network/model/user.py +++ b/sugar_network/model/user.py @@ -23,10 +23,6 @@ class User(db.Resource): def name(self, value): return value - @db.stored_property() - def color(self, value): - return value - @db.indexed_property(prefix='P', full_text=True, default='') def location(self, value): return value diff --git a/sugar_network/node/routes.py b/sugar_network/node/routes.py index 1182f28..f7fb9f4 100644 --- a/sugar_network/node/routes.py +++ b/sugar_network/node/routes.py @@ -55,12 +55,14 @@ class NodeRoutes(model.VolumeRoutes, model.FrontRoutes): self._auth_config_mtime = 0 if stats_node.stats_node.value: - self._stats = stats_node.Sniffer(volume) + stats_path = join(node.stats_root.value, 'node') + self._stats = stats_node.Sniffer(volume, stats_path) coroutine.spawn(self._commit_stats) def close(self): if self._stats is not None: self._stats.commit() + self._stats.commit_objects() @property def guid(self): @@ -402,6 +404,91 @@ class NodeRoutes(model.VolumeRoutes, model.FrontRoutes): return result +def generate_node_stats(volume, path): + tmp_path = toolkit.mkdtemp() + new_stats = stats_node.Sniffer(volume, tmp_path, True) + old_stats = stats_node.Sniffer(volume, path) + + def timeline(ts): + ts = long(ts) + end = long(time.time()) + step = None + + archives = {} + for rra in stats_node.stats_node_rras.value: + a_step, a_size = [long(i) for i in rra.split(':')[-2:]] + a_step *= stats_node.stats_node_step.value + a_start = end - min(end, a_step * a_size) + if archives.setdefault(a_start, a_step) > a_step: + archives[a_start] = a_step + archives = list(sorted(archives.items())) + + try: + while ts <= end: + while not step or archives and ts >= archives[0][0]: + archive_start, step = archives.pop(0) + ts = max(ts / step * step, archive_start) + yield ts, ts + step - 1, step + ts += step + except GeneratorExit: + shutil.rmtree(tmp_path, ignore_errors=True) + + start = next(volume['context'].find(limit=1, order_by='ctime')[0])['ctime'] + for left, right, step in timeline(start): + for resource, props in [ + ('user', []), + ('context', []), + ('implementation', ['context']), + ('artifact', ['context']), + ('feedback', ['context']), + ('solution', ['context', 'feedback']), + ('review', ['context', 'artifact', 'rating']), + ('report', ['context', 'implementation']), + ('comment', ['context', 'review', 'feedback', 'solution']), + ]: + objs, __ = volume[resource].find( + query='ctime:%s..%s' % (left, right)) + for obj in objs: + request = Request(method='POST', path=[resource], + content=obj.properties(props)) + new_stats.log(request) + for resource, props in [ + ('user', ['layer']), + ('context', ['layer']), + ('implementation', ['layer']), + ('artifact', ['layer']), + ('feedback', ['layer', 'solution']), + ('solution', ['layer']), + ('review', ['layer']), + ('report', ['layer']), + ('comment', ['layer']), + ]: + objs, __ = volume[resource].find( + query='mtime:%s..%s' % (left, right)) + for obj in objs: + if 'deleted' in obj['layer']: + request = Request(method='DELETE', + path=[resource, obj.guid]) + else: + request = Request(method='PUT', path=[resource, obj.guid], + content=obj.properties(props)) + new_stats.log(request) + downloaded = {} + for resource in ('context', 'artifact'): + stats = old_stats.report( + {resource: ['downloaded']}, left - step, right, 1) + if not stats.get(resource): + continue + stats = stats[resource][-1][1].get('downloaded') + if stats: + downloaded[resource] = {'downloaded': stats} + new_stats.commit(left + (right - left) / 2, downloaded) + + new_stats.commit_objects(True) + shutil.rmtree(path) + shutil.move(tmp_path, path) + + @contextmanager def load_bundle(volume, request, bundle_path): impl = request.copy() @@ -488,6 +575,14 @@ def load_bundle(volume, request, bundle_path): def _load_context_metadata(bundle, spec): + + def convert(data, w, h): + result = toolkit.svg_to_png(data.getvalue(), w, h) + return {'blob': result, + 'mime_type': 'image/png', + 'digest': hashlib.sha1(result.getvalue()).hexdigest(), + } + result = {} for prop in ('homepage', 'mime_types'): if spec[prop]: @@ -503,8 +598,8 @@ def _load_context_metadata(bundle, spec): 'mime_type': 'image/svg+xml', 'digest': hashlib.sha1(icon.getvalue()).hexdigest(), }, - 'preview': _svg_to_png(icon.getvalue(), 160, 120), - 'icon': _svg_to_png(icon.getvalue(), 55, 55), + 'preview': convert(icon, 160, 120), + 'icon': convert(icon, 55, 55), }) except Exception: exception(_logger, 'Failed to load icon') @@ -538,28 +633,3 @@ def _load_context_metadata(bundle, spec): exception(_logger, 'Gettext failed to read %r', mo_path[-1]) return result - - -def _svg_to_png(data, w, h): - import rsvg - import cairo - - svg = rsvg.Handle(data=data) - surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, w, h) - context = cairo.Context(surface) - - scale = min(float(w) / svg.props.width, float(h) / svg.props.height) - context.translate( - int(w - svg.props.width * scale) / 2, - int(h - svg.props.height * scale) / 2) - context.scale(scale, scale) - svg.render_cairo(context) - - result = StringIO() - surface.write_to_png(result) - result.seek(0) - - return {'blob': result, - 'mime_type': 'image/png', - 'digest': hashlib.sha1(result.getvalue()).hexdigest(), - } diff --git a/sugar_network/node/stats_node.py b/sugar_network/node/stats_node.py index 61a4a28..fb9ef5d 100644 --- a/sugar_network/node/stats_node.py +++ b/sugar_network/node/stats_node.py @@ -14,9 +14,7 @@ # along with this program. If not, see . import logging -from os.path import join -from sugar_network import node from sugar_network.toolkit.rrd import Rrd from sugar_network.toolkit import Option @@ -32,11 +30,11 @@ stats_node_step = Option( stats_node_rras = Option( 'comma separated list of RRAs for node RRD databases', default=[ - 'RRA:AVERAGE:0.5:1:288', # one day with 5min step - 'RRA:AVERAGE:0.5:3:672', # one week with 15min step - 'RRA:AVERAGE:0.5:12:744', # one month with 1h step - 'RRA:AVERAGE:0.5:144:732', # one year with 12h step - 'RRA:AVERAGE:0.5:288:36600', # hundred years with 24h step + 'RRA:AVERAGE:0.5:1:864', # 3d with 5min step + 'RRA:AVERAGE:0.5:288:3660', # 10y with 1d step + 'RRA:AVERAGE:0.5:2880:366', # 10y with 10d step + 'RRA:AVERAGE:0.5:8640:122', # 10y with 30d step + 'RRA:AVERAGE:0.5:105408:10', # 10y with 1y step ], type_cast=Option.list_cast, type_repr=Option.list_repr) @@ -45,8 +43,7 @@ _logger = logging.getLogger('node.stats_node') class Sniffer(object): - def __init__(self, volume): - path = join(node.stats_root.value, 'node') + def __init__(self, volume, path, reset=False): _logger.info('Collect node stats in %r', path) self._volume = volume @@ -54,21 +51,76 @@ class Sniffer(object): self._stats = {} for name, cls in _STATS.items(): - self._stats[name] = cls(self._stats, volume) + stats = self._stats[name] = cls(self._stats, volume, reset) + fields = {} + for attr in dir(stats): + if attr[0] == '_' or attr[0].isupper() or \ + type(getattr(stats, attr)) not in (int, long): + continue + if attr == 'total': + dst = 'GAUGE' + limit = 60 * 60 * 24 * 365 + else: + dst = 'ABSOLUTE' + limit = stats_node_step.value + fields[attr] = 'DS:%s:%s:%s:U:U' % (attr, dst, limit) + if fields: + self._rrd[name].fields = fields + + def __getitem__(self, name): + return self._rrd[name] def log(self, request): if request.cmd or request.resource not in _STATS: return self._stats[request.resource].log(request) - def commit(self, timestamp=None): + def commit(self, timestamp=None, extra_values=None): _logger.trace('Commit node stats') for resource, stats in self._stats.items(): - values = stats.commit() - if values is not None: + if resource not in self._rrd: + continue + values = {} + for field in self._rrd[resource].fields: + values[field] = getattr(stats, field) + if field != 'total': + setattr(stats, field, 0) + if extra_values and resource in extra_values: + values.update(extra_values[resource]) + if values: self._rrd[resource].put(values, timestamp=timestamp) + def commit_objects(self, reset=False): + _logger.trace('Commit object stats') + + for resource, stats in self._stats.items(): + obj = { + 'downloads': 0, + 'reviews': (0, 0), + } + directory = self._volume[resource] + for guid, obj_stats in stats.active.items(): + if not obj_stats.reviews and not obj_stats.downloads: + continue + if not directory.exists(guid): + _logger.warning('Ignore stats for missed %r %s', + guid, resource) + continue + if not reset: + obj = directory.get(guid) + patch = {} + if obj_stats.downloads: + patch['downloads'] = obj_stats.downloads + obj['downloads'] + if obj_stats.reviews: + reviews, rating = obj['reviews'] + reviews += obj_stats.reviews + rating += obj_stats.rating + patch['reviews'] = [reviews, rating] + patch['rating'] = int(round(float(rating) / reviews)) + directory.update(guid, patch) + stats.active.clear() + def report(self, dbs, start, end, records): result = {} @@ -77,11 +129,14 @@ class Sniffer(object): return result if not start: - start = min([i.first for i in rdbs]) + start = min([i.first for i in rdbs]) or 0 if not end: - end = max([i.last for i in rdbs]) + end = max([i.last for i in rdbs]) or 0 resolution = max(1, (end - start) / records) + _logger.debug('Report start=%s end=%s resolution=%s dbs=%r', + start, end, resolution, dbs) + for rdb in rdbs: info = result[rdb.name] = [] for ts, ds_values in rdb.get(start, end, resolution): @@ -105,15 +160,18 @@ class _Stats(object): RESOURCE = None OWNERS = [] - def __init__(self, stats, volume): + def __init__(self, stats, volume, reset): + self.active = {} self._stats = stats self._volume = volume - self._directory = volume[self.RESOURCE] - def log(self, request): - pass + def __getitem__(self, guid): + result = self.active.get(guid) + if result is None: + result = self.active[guid] = _ObjectStats() + return result - def commit(self): + def log(self, request): pass @@ -121,16 +179,10 @@ class _ResourceStats(_Stats): total = 0 - def __init__(self, stats, volume): - _Stats.__init__(self, stats, volume) - self.total = volume[self.RESOURCE].find(limit=0)[1] - self._active = {} - - def __getitem__(self, guid): - result = self._active.get(guid) - if result is None: - result = self._active[guid] = _ObjectStats() - return result + def __init__(self, stats, volume, reset): + _Stats.__init__(self, stats, volume, reset) + if not reset: + self.total = volume[self.RESOURCE].find(limit=0)[1] def log(self, request): if request.method == 'POST': @@ -138,37 +190,9 @@ class _ResourceStats(_Stats): elif request.method == 'DELETE': self.total -= 1 - def commit(self): - for guid, stats in self._active.items(): - if not stats.reviews and not stats.downloads: - continue - doc = self._directory.get(guid) - updates = {} - if stats.downloads: - updates['downloads'] = stats.downloads + doc['downloads'] - if stats.reviews: - reviews, rating = doc['reviews'] - reviews += stats.reviews - rating += stats.rating - updates['reviews'] = [reviews, rating] - updates['rating'] = int(round(float(rating) / reviews)) - self._directory.update(guid, updates) - self._active.clear() - - result = {} - for attr in dir(self): - if attr[0] == '_' or attr[0].isupper(): - continue - value = getattr(self, attr) - if type(value) in (set, dict): - value = len(value) - if type(value) in (int, long): - result[attr] = value - - return result - def parse_context(self, request): context = None + directory = self._volume[self.RESOURCE] def parse_context(props): for owner in self.OWNERS: @@ -186,14 +210,14 @@ class _ResourceStats(_Stats): elif self.RESOURCE == 'context': context = request.guid elif self.RESOURCE != 'user': - context = self._directory.get(request.guid)['context'] + context = directory.get(request.guid)['context'] elif request.method == 'PUT': if self.RESOURCE == 'context': context = request.guid else: context = request.content.get('context') if not context: - context = self._directory.get(request.guid)['context'] + context = directory.get(request.guid)['context'] elif request.method == 'POST': context = parse_context(request.content) @@ -211,17 +235,8 @@ class _ContextStats(_ResourceStats): released = 0 failed = 0 - reviewed = 0 downloaded = 0 - def commit(self): - result = _ResourceStats.commit(self) - self.released = 0 - self.failed = 0 - self.reviewed = 0 - self.downloaded = 0 - return result - class _ImplementationStats(_Stats): @@ -231,7 +246,7 @@ class _ImplementationStats(_Stats): def log(self, request): if request.method == 'GET': if request.prop == 'data': - context = self._directory.get(request.guid) + context = self._volume[self.RESOURCE].get(request.guid) self._stats['context'][context.context].downloads += 1 self._stats['context'].downloaded += 1 elif request.method == 'POST': @@ -253,8 +268,6 @@ class _ReviewStats(_ResourceStats): RESOURCE = 'review' OWNERS = ['artifact', 'context'] - commented = 0 - def log(self, request): _ResourceStats.log(self, request) @@ -262,17 +275,10 @@ class _ReviewStats(_ResourceStats): if request.content.get('artifact'): artifact = self._stats['artifact'] stats = artifact[request.content['artifact']] - artifact.reviewed += 1 else: stats = self._stats['context'][self.parse_context(request)] - self._stats['context'].reviewed += 1 stats.reviews += 1 - stats.rating += request.content['rating'] - - def commit(self): - result = _ResourceStats.commit(self) - self.commented = 0 - return result + stats.rating += request.content.get('rating') or 0 class _FeedbackStats(_ResourceStats): @@ -280,56 +286,18 @@ class _FeedbackStats(_ResourceStats): RESOURCE = 'feedback' OWNERS = ['context'] - solutions = 0 - commented = 0 - - def __init__(self, stats, volume): - _ResourceStats.__init__(self, stats, volume) - not_solved = volume['feedback'].find(limit=0, solution='')[1] - self.solutions = self.total - not_solved - - def log(self, request): - _ResourceStats.log(self, request) - - if request.method == 'POST': - if request.content.get('solution'): - self.solutions += 1 - elif request.method == 'PUT': - if cmp(bool(self._directory.get(request.guid)['solution']), - bool(request.content.get('solution'))): - if request.content.get('solution'): - self.solutions += 1 - else: - self.solutions -= 1 - elif request.method == 'DELETE': - if self._directory.get(request.guid)['solution']: - self.solutions -= 1 - - def commit(self): - result = _ResourceStats.commit(self) - self.commented = 0 - return result - class _SolutionStats(_ResourceStats): RESOURCE = 'solution' OWNERS = ['feedback'] - commented = 0 - - def commit(self): - result = _ResourceStats.commit(self) - self.commented = 0 - return result - class _ArtifactStats(_ResourceStats): RESOURCE = 'artifact' OWNERS = ['context'] - reviewed = 0 downloaded = 0 def log(self, request): @@ -340,25 +308,12 @@ class _ArtifactStats(_ResourceStats): self[request.guid].downloads += 1 self.downloaded += 1 - def commit(self): - result = _ResourceStats.commit(self) - self.reviewed = 0 - self.downloaded = 0 - return result - -class _CommentStats(_Stats): +class _CommentStats(_ResourceStats): RESOURCE = 'comment' OWNERS = ['solution', 'feedback', 'review'] - def log(self, request): - if request.method == 'POST': - for owner in ('solution', 'feedback', 'review'): - if request.content.get(owner): - self._stats[owner].commented += 1 - break - _STATS = {_UserStats.RESOURCE: _UserStats, _ContextStats.RESOURCE: _ContextStats, diff --git a/sugar_network/toolkit/__init__.py b/sugar_network/toolkit/__init__.py index d56ec59..cbe1b91 100644 --- a/sugar_network/toolkit/__init__.py +++ b/sugar_network/toolkit/__init__.py @@ -21,6 +21,7 @@ import shutil import logging import tempfile import collections +from cStringIO import StringIO from os.path import exists, join, islink, isdir, dirname, basename, abspath from os.path import lexists, isfile @@ -498,6 +499,27 @@ class mkdtemp(str): shutil.rmtree(self) +def svg_to_png(data, w, h): + import rsvg + import cairo + + svg = rsvg.Handle(data=data) + surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, w, h) + context = cairo.Context(surface) + + scale = min(float(w) / svg.props.width, float(h) / svg.props.height) + context.translate( + int(w - svg.props.width * scale) / 2, + int(h - svg.props.height * scale) / 2) + context.scale(scale, scale) + svg.render_cairo(context) + + result = StringIO() + surface.write_to_png(result) + result.seek(0) + return result + + def TemporaryFile(*args, **kwargs): if cachedir.value: if not exists(cachedir.value): diff --git a/sugar_network/toolkit/http.py b/sugar_network/toolkit/http.py index 2506873..d1b2fe7 100644 --- a/sugar_network/toolkit/http.py +++ b/sugar_network/toolkit/http.py @@ -310,6 +310,7 @@ class Connection(object): sys.path.insert(0, sys_path) from requests import Session, exceptions Connection._Session = Session + # pylint: disable-msg=W0601 global ConnectionError ConnectionError = exceptions.ConnectionError @@ -341,7 +342,7 @@ class SugarAuth(object): def __init__(self, key_path, profile=None): self._key_path = abspath(expanduser(key_path)) - self._profile = profile or {'color': '#000000,#000000'} + self._profile = profile or {} self._key = None self._pubkey = None self._login = None diff --git a/sugar_network/toolkit/router.py b/sugar_network/toolkit/router.py index 885bca9..6309301 100644 --- a/sugar_network/toolkit/router.py +++ b/sugar_network/toolkit/router.py @@ -575,7 +575,7 @@ class Router(object): result = None _logger.trace('%s call: request=%s response=%r result=%r', - self, request.environ, response, result) + self, request.environ, response, repr(result)[:256]) start_response(response.status, response.items()) if result_streamed: diff --git a/sugar_network/toolkit/rrd.py b/sugar_network/toolkit/rrd.py index a5f879d..da58aff 100644 --- a/sugar_network/toolkit/rrd.py +++ b/sugar_network/toolkit/rrd.py @@ -18,9 +18,10 @@ import re import os import time +import json import bisect import logging -from os.path import exists, join +from os.path import exists, join, splitext _DB_FILENAME_RE = re.compile('(.*?)(-[0-9]+){0,1}\\.rrd$') @@ -28,7 +29,7 @@ _INFO_RE = re.compile('([^[]+)\\[([^]]+)\\]\\.(.*)$') _FETCH_PAGE = 256 -_logger = logging.getLogger('sugar_stats') +_logger = logging.getLogger('rrd') _rrdtool = None @@ -76,8 +77,8 @@ class Rrd(object): def get(self, name): db = self._dbsets.get(name) if db is None: - db = self._dbsets[name] = \ - _DbSet(self._root, name, self._step, self._rras) + db = _DbSet(self._root, name, self._step, self._rras) + self._dbsets[name] = db return db @@ -89,10 +90,22 @@ class _DbSet(object): self._step = step self._rras = rras self._revisions = [] - self._field_names = [] + self._fields = None + self._field_names = None self.__db = None @property + def fields(self): + return self._field_names + + @fields.setter + def fields(self, fields): + self._field_names = fields.keys() + self._field_names.sort() + self._fields = [str(fields[i]) for i in self._field_names] + _logger.debug('Set %r fields for %r', self._fields, self.name) + + @property def first(self): if self._revisions: return self._revisions[0].first @@ -110,9 +123,11 @@ class _DbSet(object): return db def put(self, values, timestamp=None): - if not self._field_names: - self._field_names = values.keys() - self._field_names.sort() + if not self.fields: + _logger.debug('Parse fields from the first put') + self.fields = dict([ + (i, 'DS:%s:GAUGE:%s:U:U' % (i, self._step * 2)) + for i in values]) if not timestamp: timestamp = int(time.time()) @@ -133,7 +148,7 @@ class _DbSet(object): _logger.debug('Put %r to %s', value, db.path) - db.put(':'.join(value)) + db.put(':'.join(value), timestamp) def get(self, start=None, end=None, resolution=None): if not self._revisions: @@ -154,11 +169,11 @@ class _DbSet(object): break start = start - start % self._step - self._step - end = min(end, start + _FETCH_PAGE * resolution) - end -= end % self._step + self._step + last = min(end, start + _FETCH_PAGE * resolution) + last -= last % self._step + self._step for db in reversed(revisions): - db_end = min(end, db.last - self._step) + db_end = min(last, db.last - self._step) if start > db_end: break (row_start, start, row_step), __, rows = _rrdtool.fetch( @@ -169,17 +184,16 @@ class _DbSet(object): '--resolution', str(resolution)) for raw_row in rows: row_start += row_step + if row_start > end: + break row = {} - accept = False for i, value in enumerate(raw_row): - row[db.field_names[i]] = value - accept = accept or value is not None - if accept: - yield row_start, row + row[db.field_names[i]] = value or .0 + yield row_start, row start = db_end + 1 def _get_db(self, timestamp): - if self.__db is None and self._field_names: + if self.__db is None and self._fields: if self._revisions: db = self._revisions[-1] if db.last >= timestamp: @@ -189,14 +203,13 @@ class _DbSet(object): return None if db.step != self._step or db.rras != self._rras or \ db.field_names != self._field_names: - db = self._create_db(self._field_names, db.revision + 1, - db.last) + db = self._create_db(db.revision + 1, db.last) else: - db = self._create_db(self._field_names, 0, timestamp) + db = self._create_db(0, timestamp) self.__db = db return self.__db - def _create_db(self, field_names, revision, timestamp): + def _create_db(self, revision, timestamp): filename = self.name if revision: filename += '-%s' % revision @@ -205,15 +218,11 @@ class _DbSet(object): _logger.debug('Create %s database in %s start=%s step=%s', filename, self._root, timestamp, self._step) - fields = [] - for name in field_names: - fields.append(str('DS:%s:GAUGE:%s:U:U' % (name, self._step * 2))) - _rrdtool.create( str(join(self._root, filename)), '--start', str(timestamp - self._step), '--step', str(self._step), - *(fields + self._rras)) + *(self._fields + self._rras)) return self.load(filename, revision) @@ -222,6 +231,7 @@ class _Db(object): def __init__(self, path, revision=0): self.path = str(path) + self._meta_path = splitext(path)[0] + '.meta' self.revision = revision self.fields = [] self.field_names = [] @@ -229,6 +239,7 @@ class _Db(object): info = _rrdtool.info(self.path) self.step = info['step'] + self.first = 0 self.last = info['last_update'] fields = {} @@ -257,13 +268,17 @@ class _Db(object): self.fields.append(props) self.field_names.append(name) - def put(self, value): + if exists(self._meta_path): + with file(self._meta_path) as f: + self.first = json.load(f).get('first') + + def put(self, value, timestamp): + if not self.first: + with file(self._meta_path, 'w') as f: + json.dump({'first': timestamp}, f) + self.first = timestamp _rrdtool.update(self.path, str(value)) self.last = _rrdtool.info(self.path)['last_update'] - @property - def first(self): - return _rrdtool.first(self.path) - def __cmp__(self, other): return cmp(self.revision, other.revision) diff --git a/tests/__init__.py b/tests/__init__.py index 58203a9..559020f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -130,7 +130,6 @@ class Test(unittest.TestCase): if tmp_root is None: self.override(_Auth, 'profile', lambda self: { 'name': 'test', - 'color': '#000000,#000000', 'pubkey': PUBKEY, }) diff --git a/tests/units/db/routes.py b/tests/units/db/routes.py index 4dd0fea..95aeb4e 100755 --- a/tests/units/db/routes.py +++ b/tests/units/db/routes.py @@ -1180,7 +1180,7 @@ class RoutesTest(tests.Test): {'user': {'role': 2, 'order': 0}}, self.volume['document'].get(guid)['author']) - self.volume['user'].create({'guid': 'user', 'color': '', 'pubkey': '', 'name': 'User'}) + self.volume['user'].create({'guid': 'user', 'pubkey': '', 'name': 'User'}) guid = self.call('POST', ['document'], content={}, principal='user') self.assertEqual( @@ -1203,9 +1203,9 @@ class RoutesTest(tests.Test): self.volume = db.Volume('db', [User, Document]) - self.volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'UserName1'}) - self.volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User Name2'}) - self.volume['user'].create({'guid': 'user3', 'color': '', 'pubkey': '', 'name': 'User Name 3'}) + self.volume['user'].create({'guid': 'user1', 'pubkey': '', 'name': 'UserName1'}) + self.volume['user'].create({'guid': 'user2', 'pubkey': '', 'name': 'User Name2'}) + self.volume['user'].create({'guid': 'user3', 'pubkey': '', 'name': 'User Name 3'}) guid1 = self.call('POST', ['document'], content={}, principal='user1') guid2 = self.call('POST', ['document'], content={}, principal='user2') @@ -1245,9 +1245,9 @@ class RoutesTest(tests.Test): self.volume = db.Volume('db', [User, Document]) - self.volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'}) - self.volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User2'}) - self.volume['user'].create({'guid': 'user3', 'color': '', 'pubkey': '', 'name': 'User3'}) + self.volume['user'].create({'guid': 'user1', 'pubkey': '', 'name': 'User1'}) + self.volume['user'].create({'guid': 'user2', 'pubkey': '', 'name': 'User2'}) + self.volume['user'].create({'guid': 'user3', 'pubkey': '', 'name': 'User3'}) guid = self.call('POST', ['document'], content={}, principal='user1') self.call('PUT', ['document', guid], cmd='useradd', user='user2', role=0) @@ -1326,7 +1326,7 @@ class RoutesTest(tests.Test): pass self.volume = db.Volume('db', [User, Document]) - self.volume['user'].create({'guid': 'user', 'color': '', 'pubkey': '', 'name': 'User'}) + self.volume['user'].create({'guid': 'user', 'pubkey': '', 'name': 'User'}) guid1 = self.call('POST', ['document'], content={}, principal='user') self.assertEqual({'user': {'name': 'User', 'role': 3, 'order': 0}}, self.volume['document'].get(guid1)['author']) @@ -1350,8 +1350,8 @@ class RoutesTest(tests.Test): self.volume = db.Volume('db', [User, Document]) - self.volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'}) - self.volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User2'}) + self.volume['user'].create({'guid': 'user1', 'pubkey': '', 'name': 'User1'}) + self.volume['user'].create({'guid': 'user2', 'pubkey': '', 'name': 'User2'}) guid = self.call('POST', ['document'], content={}, principal='user1') self.assertEqual([ @@ -1418,7 +1418,7 @@ class RoutesTest(tests.Test): self.volume = db.Volume('db', [User, Document]) - self.volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'}) + self.volume['user'].create({'guid': 'user1', 'pubkey': '', 'name': 'User1'}) guid = self.call('POST', ['document'], content={}, principal='user1') self.call('PUT', ['document', guid], cmd='useradd', user='User2', role=0) @@ -1470,8 +1470,8 @@ class RoutesTest(tests.Test): self.volume = db.Volume('db', [User, Document]) - self.volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'}) - self.volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User2'}) + self.volume['user'].create({'guid': 'user1', 'pubkey': '', 'name': 'User1'}) + self.volume['user'].create({'guid': 'user2', 'pubkey': '', 'name': 'User2'}) guid = self.call('POST', ['document'], content={}, principal='user1') self.call('PUT', ['document', guid], cmd='useradd', user='user2') self.call('PUT', ['document', guid], cmd='useradd', user='User3') diff --git a/tests/units/node/node.py b/tests/units/node/node.py index b5abe78..9d6ae6f 100755 --- a/tests/units/node/node.py +++ b/tests/units/node/node.py @@ -20,7 +20,7 @@ from sugar_network.client import Connection, keyfile from sugar_network.toolkit import http, coroutine from sugar_network.toolkit.rrd import Rrd from sugar_network.node import stats_user, stats_node, obs -from sugar_network.node.routes import NodeRoutes +from sugar_network.node.routes import NodeRoutes, generate_node_stats from sugar_network.node.master import MasterRoutes from sugar_network.model.user import User from sugar_network.model.context import Context @@ -48,7 +48,6 @@ class NodeTest(tests.Test): call(cp, method='POST', document='user', principal=tests.UID, content={ 'name': 'user', - 'color': '', 'pubkey': tests.PUBKEY, }) @@ -129,6 +128,7 @@ class NodeTest(tests.Test): self.assertEqual({ 'user': [ + (ts + 0, {'total': 0.0}), (ts + 3, {'total': 2.0}), (ts + 6, {'total': 5.0}), (ts + 9, {'total': 8.0}), @@ -165,7 +165,7 @@ class NodeTest(tests.Test): def test_HandleDeletes(self): volume = db.Volume('db', model.RESOURCES) - volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) cp = NodeRoutes('guid', volume) guid = call(cp, method='POST', document='context', principal=tests.UID, content={ @@ -200,7 +200,7 @@ class NodeTest(tests.Test): def test_SimulateDeleteEvents(self): volume = db.Volume('db', model.RESOURCES) - volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) cp = NodeRoutes('guid', volume) guid = call(cp, method='POST', document='context', principal=tests.UID, content={ @@ -226,7 +226,6 @@ class NodeTest(tests.Test): guid = call(cp, method='POST', document='user', principal=tests.UID2, content={ 'name': 'user', - 'color': '', 'pubkey': tests.PUBKEY, }) assert guid is None @@ -234,7 +233,7 @@ class NodeTest(tests.Test): def test_UnauthorizedCommands(self): volume = db.Volume('db', model.RESOURCES) - volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) class Routes(NodeRoutes): @@ -272,7 +271,7 @@ class NodeTest(tests.Test): pass volume = db.Volume('db', [User, Document]) - volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) cp = Routes('guid', volume) guid = call(cp, method='POST', document='document', principal=tests.UID, content={}) @@ -287,7 +286,6 @@ class NodeTest(tests.Test): call(cp, method='POST', document='user', principal=tests.UID2, content={ 'name': 'user1', - 'color': '', 'pubkey': tests.PUBKEY, }) self.assertEqual('user1', call(cp, method='GET', document='user', guid=tests.UID, prop='name')) @@ -310,8 +308,8 @@ class NodeTest(tests.Test): return 'ok' volume = db.Volume('db', [User]) - volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) - volume['user'].create({'guid': tests.UID2, 'name': 'test', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY2)}}) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) + volume['user'].create({'guid': tests.UID2, 'name': 'test', 'pubkey': {'blob': StringIO(tests.PUBKEY2)}}) cp = Routes('guid', volume) self.assertRaises(http.Forbidden, call, cp, method='PROBE') @@ -327,8 +325,8 @@ class NodeTest(tests.Test): return value volume = db.Volume('db', [User, Document]) - volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) - volume['user'].create({'guid': tests.UID2, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY2)}}) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) + volume['user'].create({'guid': tests.UID2, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY2)}}) cp = NodeRoutes('guid', volume) guid = call(cp, method='POST', document='document', principal=tests.UID, content={'prop': '1'}) @@ -348,8 +346,8 @@ class NodeTest(tests.Test): return value volume = db.Volume('db', [User, Document]) - volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) - volume['user'].create({'guid': tests.UID2, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY2)}}) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) + volume['user'].create({'guid': tests.UID2, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY2)}}) cp = NodeRoutes('guid', volume) guid = call(cp, method='POST', document='document', principal=tests.UID, content={'prop': '1'}) @@ -369,7 +367,7 @@ class NodeTest(tests.Test): pass volume = db.Volume('db', [User]) - volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) cp = Routes('guid', volume) self.assertRaises(http.Forbidden, call, cp, 'PROBE', principal=tests.UID) @@ -407,7 +405,7 @@ class NodeTest(tests.Test): def test_SetUser(self): volume = db.Volume('db', model.RESOURCES) - volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) cp = NodeRoutes('guid', volume) guid = call(cp, method='POST', document='context', principal=tests.UID, content={ @@ -422,7 +420,7 @@ class NodeTest(tests.Test): def test_find_MaxLimit(self): volume = db.Volume('db', model.RESOURCES) - volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) cp = NodeRoutes('guid', volume) call(cp, method='POST', document='context', principal=tests.UID, content={ @@ -453,7 +451,7 @@ class NodeTest(tests.Test): def test_DeletedDocuments(self): volume = db.Volume('db', model.RESOURCES) - volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) cp = NodeRoutes('guid', volume) guid = call(cp, method='POST', document='context', principal=tests.UID, content={ @@ -474,7 +472,7 @@ class NodeTest(tests.Test): def test_CreateGUID(self): # TODO Temporal security hole, see TODO volume = db.Volume('db', model.RESOURCES) - volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) cp = NodeRoutes('guid', volume) call(cp, method='POST', document='context', principal=tests.UID, content={ 'guid': 'foo', @@ -489,7 +487,7 @@ class NodeTest(tests.Test): def test_CreateMalformedGUID(self): volume = db.Volume('db', model.RESOURCES) - volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) cp = MasterRoutes('guid', volume) self.assertRaises(RuntimeError, call, cp, method='POST', document='context', principal=tests.UID, content={ @@ -502,7 +500,7 @@ class NodeTest(tests.Test): def test_FailOnExistedGUID(self): volume = db.Volume('db', model.RESOURCES) - volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) + volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}}) cp = MasterRoutes('guid', volume) guid = call(cp, method='POST', document='context', principal=tests.UID, content={ @@ -935,6 +933,551 @@ class NodeTest(tests.Test): self.assertEqual(sorted(['origin', 'deleted']), sorted(volume['implementation'].get(impl2)['layer'])) self.assertEqual([], volume['implementation'].get(impl3)['layer']) + def test_generate_node_stats_Posts(self): + node.stats_root.value = 'stats' + stats_node.stats_node.value = True + stats_node.stats_node_rras.value = ['RRA:AVERAGE:0.5:1:10', 'RRA:AVERAGE:0.5:10:10'] + volume = db.Volume('db', model.RESOURCES) + + ts = 1000000000 + + volume['user'].create({ + 'guid': 'user_1', + 'ctime': ts + 1, + 'mtime': ts + 1, + 'layer': [], + 'name': '', + }) + volume['context'].create({ + 'guid': 'context_1', + 'ctime': ts + 1, + 'mtime': ts + 1, + 'layer': [], + 'type': 'activity', + 'title': '', + 'summary': '', + 'description': '', + }) + volume['implementation'].create({ + 'guid': 'impl_1', + 'ctime': ts + 2, + 'mtime': ts + 2, + 'layer': [], + 'context': 'context_1', + 'license': ['GPL-3'], + 'version': '1', + }) + volume['artifact'].create({ + 'guid': 'artifact_1', + 'ctime': ts + 3, + 'mtime': ts + 3, + 'layer': [], + 'context': 'context_1', + 'type': 'preview', + 'title': '', + 'description': '', + }) + volume['feedback'].create({ + 'guid': 'feedback_1', + 'ctime': ts + 4, + 'mtime': ts + 4, + 'layer': [], + 'context': 'context_1', + 'title': '', + 'content': '', + 'type': 'idea', + }) + volume['solution'].create({ + 'guid': 'solution_1', + 'ctime': ts + 5, + 'mtime': ts + 5, + 'layer': [], + 'context': 'context_1', + 'feedback': 'feedback_1', + 'content': '', + }) + volume['review'].create({ + 'guid': 'review_1', + 'ctime': ts + 6, + 'mtime': ts + 6, + 'layer': [], + 'context': 'context_1', + 'rating': 1, + 'title': '', + 'content': '', + }) + volume['review'].create({ + 'guid': 'review_2', + 'ctime': ts + 6, + 'mtime': ts + 6, + 'layer': [], + 'context': 'context_1', + 'artifact': 'artifact_1', + 'rating': 2, + 'title': '', + 'content': '', + }) + volume['comment'].create({ + 'guid': 'comment_1', + 'ctime': ts + 7, + 'mtime': ts + 7, + 'layer': [], + 'context': 'context_1', + 'review': 'review_1', + 'message': '', + }) + volume['comment'].create({ + 'guid': 'comment_2', + 'ctime': ts + 7, + 'mtime': ts + 7, + 'layer': [], + 'context': 'context_1', + 'feedback': 'feedback_1', + 'message': '', + }) + volume['comment'].create({ + 'guid': 'comment_3', + 'ctime': ts + 7, + 'mtime': ts + 7, + 'layer': [], + 'context': 'context_1', + 'solution': 'solution_1', + 'message': '', + }) + volume['report'].create({ + 'guid': 'report_1', + 'ctime': ts + 8, + 'mtime': ts + 8, + 'layer': [], + 'context': 'context_1', + 'implementation': 'impl_1', + 'error': '', + }) + + volume['user'].create({ + 'guid': 'user_2', + 'ctime': ts + 4, + 'mtime': ts + 4, + 'layer': [], + 'name': '', + }) + volume['context'].create({ + 'guid': 'context_2', + 'ctime': ts + 4, + 'mtime': ts + 4, + 'layer': [], + 'type': 'activity', + 'title': '', + 'summary': '', + 'description': '', + }) + volume['implementation'].create({ + 'guid': 'impl_2', + 'ctime': ts + 4, + 'mtime': ts + 4, + 'layer': [], + 'context': 'context_2', + 'license': ['GPL-3'], + 'version': '1', + }) + volume['implementation'].create({ + 'guid': 'impl_3', + 'ctime': ts + 4, + 'mtime': ts + 4, + 'layer': [], + 'context': 'context_2', + 'license': ['GPL-3'], + 'version': '1', + }) + volume['review'].create({ + 'guid': 'review_3', + 'ctime': ts + 4, + 'mtime': ts + 4, + 'layer': [], + 'context': 'context_2', + 'rating': 3, + 'title': '', + 'content': '', + }) + volume['review'].create({ + 'guid': 'review_4', + 'ctime': ts + 4, + 'mtime': ts + 4, + 'layer': [], + 'context': 'context_2', + 'rating': 4, + 'title': '', + 'content': '', + }) + volume['report'].create({ + 'guid': 'report_2', + 'ctime': ts + 4, + 'mtime': ts + 4, + 'layer': [], + 'context': 'context_2', + 'implementation': 'impl_1', + 'error': '', + }) + volume['report'].create({ + 'guid': 'report_3', + 'ctime': ts + 4, + 'mtime': ts + 4, + 'layer': [], + 'context': 'context_2', + 'implementation': 'impl_1', + 'error': '', + }) + volume['artifact'].create({ + 'guid': 'artifact_2', + 'ctime': ts + 4, + 'mtime': ts + 4, + 'layer': [], + 'context': 'context_2', + 'type': 'preview', + 'title': '', + 'description': '', + }) + volume['feedback'].create({ + 'guid': 'feedback_2', + 'ctime': ts + 4, + 'mtime': ts + 4, + 'layer': [], + 'context': 'context_2', + 'title': '', + 'content': '', + 'type': 'idea', + }) + volume['solution'].create({ + 'guid': 'solution_2', + 'ctime': ts + 4, + 'mtime': ts + 4, + 'layer': [], + 'context': 'context_2', + 'feedback': 'feedback_2', + 'content': '', + }) + volume['comment'].create({ + 'guid': 'comment_4', + 'ctime': ts + 4, + 'mtime': ts + 4, + 'layer': [], + 'context': 'context_2', + 'review': 'review_3', + 'message': '', + }) + volume['comment'].create({ + 'guid': 'comment_5', + 'ctime': ts + 4, + 'mtime': ts + 4, + 'layer': [], + 'context': 'context_2', + 'review': 'review_4', + 'message': '', + }) + volume['comment'].create({ + 'guid': 'comment_6', + 'ctime': ts + 4, + 'mtime': ts + 4, + 'layer': [], + 'context': 'context_2', + 'feedback': 'feedback_2', + 'message': '', + }) + volume['comment'].create({ + 'guid': 'comment_7', + 'ctime': ts + 4, + 'mtime': ts + 4, + 'layer': [], + 'context': 'context_2', + 'solution': 'solution_2', + 'message': '', + }) + + self.override(time, 'time', lambda: ts + 9) + old_stats = stats_node.Sniffer(volume, 'stats/node') + old_stats.log(Request(method='GET', path=['implementation', 'impl_1', 'data'])) + old_stats.log(Request(method='GET', path=['artifact', 'artifact_1', 'data'])) + old_stats.commit(ts + 1) + old_stats.commit_objects() + old_stats.commit(ts + 2) + old_stats.commit(ts + 3) + old_stats.log(Request(method='GET', path=['implementation', 'impl_1', 'data'])) + old_stats.log(Request(method='GET', path=['implementation', 'impl_2', 'data'])) + old_stats.commit(ts + 4) + old_stats.commit_objects() + old_stats.commit(ts + 5) + old_stats.commit(ts + 6) + old_stats.log(Request(method='GET', path=['artifact', 'artifact_1', 'data'])) + old_stats.log(Request(method='GET', path=['artifact', 'artifact_2', 'data'])) + old_stats.commit(ts + 7) + old_stats.commit_objects() + old_stats.commit(ts + 8) + old_stats.commit_objects() + + generate_node_stats(volume, 'stats/node') + cp = NodeRoutes('guid', volume) + + self.assertEqual({ + 'user': [ + (ts + 1, {'total': 1.0}), + (ts + 2, {'total': 1.0}), + (ts + 3, {'total': 1.0}), + (ts + 4, {'total': 2.0}), + (ts + 5, {'total': 2.0}), + (ts + 6, {'total': 2.0}), + (ts + 7, {'total': 2.0}), + (ts + 8, {'total': 2.0}), + (ts + 9, {'total': 2.0}), + ], + 'context': [ + (ts + 1, {'total': 1.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 1.0}), + (ts + 2, {'total': 1.0, 'released': 1.0, 'failed': 0.0, 'downloaded': 0.0}), + (ts + 3, {'total': 1.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}), + (ts + 4, {'total': 2.0, 'released': 2.0, 'failed': 2.0, 'downloaded': 2.0}), + (ts + 5, {'total': 2.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}), + (ts + 6, {'total': 2.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}), + (ts + 7, {'total': 2.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}), + (ts + 8, {'total': 2.0, 'released': 0.0, 'failed': 1.0, 'downloaded': 0.0}), + (ts + 9, {'total': 2.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}), + ], + 'review': [ + (ts + 1, {'total': 0.0}), + (ts + 2, {'total': 0.0}), + (ts + 3, {'total': 0.0}), + (ts + 4, {'total': 2.0}), + (ts + 5, {'total': 2.0}), + (ts + 6, {'total': 4.0}), + (ts + 7, {'total': 4.0}), + (ts + 8, {'total': 4.0}), + (ts + 9, {'total': 4.0}), + ], + 'feedback': [ + (ts + 1, {'total': 0.0}), + (ts + 2, {'total': 0.0}), + (ts + 3, {'total': 0.0}), + (ts + 4, {'total': 2.0}), + (ts + 5, {'total': 2.0}), + (ts + 6, {'total': 2.0}), + (ts + 7, {'total': 2.0}), + (ts + 8, {'total': 2.0}), + (ts + 9, {'total': 2.0}), + ], + 'solution': [ + (ts + 1, {'total': 0.0}), + (ts + 2, {'total': 0.0}), + (ts + 3, {'total': 0.0}), + (ts + 4, {'total': 1.0}), + (ts + 5, {'total': 2.0}), + (ts + 6, {'total': 2.0}), + (ts + 7, {'total': 2.0}), + (ts + 8, {'total': 2.0}), + (ts + 9, {'total': 2.0}), + ], + 'artifact': [ + (ts + 1, {'total': 0.0, 'downloaded': 1.0}), + (ts + 2, {'total': 0.0, 'downloaded': 0.0}), + (ts + 3, {'total': 1.0, 'downloaded': 0.0}), + (ts + 4, {'total': 2.0, 'downloaded': 0.0}), + (ts + 5, {'total': 2.0, 'downloaded': 0.0}), + (ts + 6, {'total': 2.0, 'downloaded': 0.0}), + (ts + 7, {'total': 2.0, 'downloaded': 2.0}), + (ts + 8, {'total': 2.0, 'downloaded': 0.0}), + (ts + 9, {'total': 2.0, 'downloaded': 0.0}), + ], + 'comment': [ + (ts + 1, {'total': 0.0}), + (ts + 2, {'total': 0.0}), + (ts + 3, {'total': 0.0}), + (ts + 4, {'total': 4.0}), + (ts + 5, {'total': 4.0}), + (ts + 6, {'total': 4.0}), + (ts + 7, {'total': 7.0}), + (ts + 8, {'total': 7.0}), + (ts + 9, {'total': 7.0}), + ], + }, + call(cp, method='GET', cmd='stats', source=[ + 'user.total', + 'context.total', + 'context.released', + 'context.failed', + 'context.downloaded', + 'review.total', + 'feedback.total', + 'solution.total', + 'artifact.total', + 'artifact.downloaded', + 'comment.total', + ], start=ts + 1, end=ts + 10)) + + self.assertEqual({ + 'downloads': 2, + 'rating': 1, + 'reviews': [1, 1], + }, + volume['context'].get('context_1').properties(['downloads', 'rating', 'reviews'])) + self.assertEqual({ + 'downloads': 1, + 'rating': 4, + 'reviews': [2, 7], + }, + volume['context'].get('context_2').properties(['downloads', 'rating', 'reviews'])) + self.assertEqual({ + 'downloads': 2, + 'rating': 2, + 'reviews': [1, 2], + }, + volume['artifact'].get('artifact_1').properties(['downloads', 'rating', 'reviews'])) + self.assertEqual({ + 'downloads': 1, + 'rating': 0, + 'reviews': [0, 0], + }, + volume['artifact'].get('artifact_2').properties(['downloads', 'rating', 'reviews'])) + + def test_generate_node_stats_Deletes(self): + node.stats_root.value = 'stats' + stats_node.stats_node.value = True + stats_node.stats_node_rras.value = ['RRA:AVERAGE:0.5:1:10', 'RRA:AVERAGE:0.5:10:10'] + volume = db.Volume('db', model.RESOURCES) + + ts = 1000000000 + + volume['user'].create({ + 'guid': 'user_1', + 'ctime': ts + 1, + 'mtime': ts + 2, + 'layer': ['deleted'], + 'name': '', + }) + volume['context'].create({ + 'guid': 'context_1', + 'ctime': ts + 1, + 'mtime': ts + 2, + 'layer': ['deleted'], + 'type': 'activity', + 'title': '', + 'summary': '', + 'description': '', + }) + volume['implementation'].create({ + 'guid': 'impl_1', + 'ctime': ts + 1, + 'mtime': ts + 2, + 'layer': ['deleted'], + 'context': 'context_1', + 'license': ['GPL-3'], + 'version': '1', + }) + volume['artifact'].create({ + 'guid': 'artifact_1', + 'ctime': ts + 1, + 'mtime': ts + 2, + 'layer': ['deleted'], + 'context': 'context_1', + 'type': 'preview', + 'title': '', + 'description': '', + }) + volume['feedback'].create({ + 'guid': 'feedback_1', + 'ctime': ts + 1, + 'mtime': ts + 2, + 'layer': ['deleted'], + 'context': 'context_1', + 'title': '', + 'content': '', + 'type': 'idea', + }) + volume['solution'].create({ + 'guid': 'solution_1', + 'ctime': ts + 1, + 'mtime': ts + 2, + 'layer': ['deleted'], + 'context': 'context_1', + 'feedback': 'feedback_1', + 'content': '', + }) + volume['review'].create({ + 'guid': 'review_1', + 'ctime': ts + 1, + 'mtime': ts + 2, + 'layer': ['deleted'], + 'context': 'context_1', + 'rating': 1, + 'title': '', + 'content': '', + }) + volume['comment'].create({ + 'guid': 'comment_1', + 'ctime': ts + 1, + 'mtime': ts + 2, + 'layer': ['deleted'], + 'context': 'context_1', + 'review': 'review_1', + 'message': '', + }) + volume['report'].create({ + 'guid': 'report_1', + 'ctime': ts + 1, + 'mtime': ts + 2, + 'layer': ['deleted'], + 'context': 'context_1', + 'implementation': 'impl_1', + 'error': '', + }) + + self.override(time, 'time', lambda: ts + 9) + generate_node_stats(volume, 'stats/node') + cp = NodeRoutes('guid', volume) + + self.assertEqual({ + 'user': [ + (ts + 1, {'total': 1.0}), + (ts + 2, {'total': 0.0}), + (ts + 3, {'total': 0.0}), + ], + 'context': [ + (ts + 1, {'total': 1.0}), + (ts + 2, {'total': 0.0}), + (ts + 3, {'total': 0.0}), + ], + 'review': [ + (ts + 1, {'total': 1.0}), + (ts + 2, {'total': 0.0}), + (ts + 3, {'total': 0.0}), + ], + 'feedback': [ + (ts + 1, {'total': 1.0}), + (ts + 2, {'total': 0.0}), + (ts + 3, {'total': 0.0}), + ], + 'solution': [ + (ts + 1, {'total': 1.0}), + (ts + 2, {'total': 0.0}), + (ts + 3, {'total': 0.0}), + ], + 'artifact': [ + (ts + 1, {'total': 1.0}), + (ts + 2, {'total': 0.0}), + (ts + 3, {'total': 0.0}), + ], + 'comment': [ + (ts + 1, {'total': 1.0}), + (ts + 2, {'total': 0.0}), + (ts + 3, {'total': 0.0}), + ], + }, + call(cp, method='GET', cmd='stats', source=[ + 'user.total', + 'context.total', + 'review.total', + 'feedback.total', + 'solution.total', + 'artifact.total', + 'comment.total', + ], start=ts + 1, end=ts + 3)) + def call(routes, method, document=None, guid=None, prop=None, principal=None, content=None, **kwargs): path = [''] diff --git a/tests/units/node/stats_node.py b/tests/units/node/stats_node.py index a0ae5d0..da2f3e7 100755 --- a/tests/units/node/stats_node.py +++ b/tests/units/node/stats_node.py @@ -16,16 +16,15 @@ class StatsTest(tests.Test): def test_InitializeTotals(self): volume = db.Volume('local', model.RESOURCES) - stats = Sniffer(volume) + stats = Sniffer(volume, 'stats/node') self.assertEqual(0, stats._stats['user'].total) self.assertEqual(0, stats._stats['context'].total) self.assertEqual(0, stats._stats['review'].total) self.assertEqual(0, stats._stats['feedback'].total) - self.assertEqual(0, stats._stats['feedback'].solutions) self.assertEqual(0, stats._stats['solution'].total) self.assertEqual(0, stats._stats['artifact'].total) - volume['user'].create({'guid': 'user', 'name': 'user', 'color': '', 'pubkey': ''}) + volume['user'].create({'guid': 'user', 'name': 'user', 'pubkey': ''}) volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) volume['review'].create({'guid': 'review', 'context': 'context', 'title': '', 'content': '', 'rating': 5}) volume['feedback'].create({'guid': 'feedback', 'context': 'context', 'type': 'idea', 'title': '', 'content': ''}) @@ -33,18 +32,17 @@ class StatsTest(tests.Test): volume['solution'].create({'guid': 'solution', 'context': 'context', 'feedback': 'feedback', 'content': ''}) volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''}) - stats = Sniffer(volume) + stats = Sniffer(volume, 'stats/node') self.assertEqual(1, stats._stats['user'].total) self.assertEqual(1, stats._stats['context'].total) self.assertEqual(1, stats._stats['review'].total) self.assertEqual(2, stats._stats['feedback'].total) - self.assertEqual(1, stats._stats['feedback'].solutions) self.assertEqual(1, stats._stats['solution'].total) self.assertEqual(1, stats._stats['artifact'].total) def test_POSTs(self): volume = db.Volume('local', model.RESOURCES) - stats = Sniffer(volume) + stats = Sniffer(volume, 'stats/node') request = Request(method='POST', path=['context']) request.principal = 'user' @@ -55,7 +53,7 @@ class StatsTest(tests.Test): def test_DELETEs(self): volume = db.Volume('local', model.RESOURCES) - stats = Sniffer(volume) + stats = Sniffer(volume, 'stats/node') request = Request(method='DELETE', path=['context']) request.principal = 'user' @@ -64,33 +62,9 @@ class StatsTest(tests.Test): stats.log(request) self.assertEqual(-3, stats._stats['context'].total) - def test_FeedbackSolutions(self): - volume = db.Volume('local', model.RESOURCES) - stats = Sniffer(volume) - volume['feedback'].create({'guid': 'guid', 'context': 'context', 'type': 'idea', 'title': '', 'content': ''}) - - request = Request(method='PUT', path=['feedback', 'guid']) - request.principal = 'user' - request.content = {} - stats.log(request) - self.assertEqual(0, stats._stats['feedback'].solutions) - - request.content = {'solution': 'solution'} - stats.log(request) - self.assertEqual(1, stats._stats['feedback'].solutions) - - request.content = {'solution': None} - stats.log(request) - self.assertEqual(1, stats._stats['feedback'].solutions) - - volume['feedback'].update('guid', {'solution': 'exists'}) - request.content = {'solution': None} - stats.log(request) - self.assertEqual(0, stats._stats['feedback'].solutions) - def test_Comments(self): volume = db.Volume('local', model.RESOURCES) - stats = Sniffer(volume) + stats = Sniffer(volume, 'stats/node') volume['solution'].create({'guid': 'solution', 'context': 'context', 'feedback': 'feedback', 'content': ''}) volume['feedback'].create({'guid': 'feedback', 'context': 'context', 'type': 'idea', 'title': '', 'content': ''}) volume['review'].create({'guid': 'review', 'context': 'context', 'title': '', 'content': '', 'rating': 5}) @@ -99,50 +73,51 @@ class StatsTest(tests.Test): request.principal = 'user' request.content = {'solution': 'solution'} stats.log(request) - self.assertEqual(1, stats._stats['solution'].commented) + self.assertEqual(1, stats._stats['comment'].total) request = Request(method='POST', path=['comment']) request.principal = 'user' request.content = {'feedback': 'feedback'} stats.log(request) - self.assertEqual(1, stats._stats['feedback'].commented) + self.assertEqual(2, stats._stats['comment'].total) request = Request(method='POST', path=['comment']) request.principal = 'user' request.content = {'review': 'review'} stats.log(request) - self.assertEqual(1, stats._stats['review'].commented) + self.assertEqual(3, stats._stats['comment'].total) def test_Reviewes(self): volume = db.Volume('local', model.RESOURCES) - stats = Sniffer(volume) + stats = Sniffer(volume, 'stats/node') volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''}) request = Request(method='POST', path=['review']) request.principal = 'user' - request.content = {'context': 'context', 'rating': 0} + request.content = {'context': 'context', 'rating': 1} stats.log(request) - self.assertEqual(1, stats._stats['context'].reviewed) - self.assertEqual(0, stats._stats['artifact'].reviewed) + self.assertEqual(1, stats._stats['review'].total) request = Request(method='POST', path=['review']) request.principal = 'user' - request.content = {'context': 'context', 'artifact': '', 'rating': 0} + request.content = {'context': 'context', 'artifact': '', 'rating': 2} stats.log(request) - self.assertEqual(2, stats._stats['context'].reviewed) - self.assertEqual(0, stats._stats['artifact'].reviewed) + self.assertEqual(2, stats._stats['review'].total) request = Request(method='POST', path=['review']) request.principal = 'user' - request.content = {'artifact': 'artifact', 'rating': 0} + request.content = {'artifact': 'artifact', 'rating': 3} stats.log(request) - self.assertEqual(2, stats._stats['context'].reviewed) - self.assertEqual(1, stats._stats['artifact'].reviewed) + self.assertEqual(3, stats._stats['review'].total) + + stats.commit_objects() + self.assertEqual([2, 3], volume['context'].get('context')['reviews']) + self.assertEqual([1, 3], volume['artifact'].get('artifact')['reviews']) def test_ContextDownloaded(self): volume = db.Volume('local', model.RESOURCES) - stats = Sniffer(volume) + stats = Sniffer(volume, 'stats/node') volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) volume['implementation'].create({'guid': 'implementation', 'context': 'context', 'license': 'GPLv3', 'version': '1', 'date': 0, 'stability': 'stable', 'notes': ''}) @@ -158,7 +133,7 @@ class StatsTest(tests.Test): def test_ContextReleased(self): volume = db.Volume('local', model.RESOURCES) - stats = Sniffer(volume) + stats = Sniffer(volume, 'stats/node') volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) request = Request(method='POST', path=['implementation']) @@ -169,7 +144,7 @@ class StatsTest(tests.Test): def test_ContextFailed(self): volume = db.Volume('local', model.RESOURCES) - stats = Sniffer(volume) + stats = Sniffer(volume, 'stats/node') volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) request = Request(method='POST', path=['report']) @@ -180,7 +155,7 @@ class StatsTest(tests.Test): def test_ArtifactDownloaded(self): volume = db.Volume('local', model.RESOURCES) - stats = Sniffer(volume) + stats = Sniffer(volume, 'stats/node') volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''}) request = Request(method='GET', path=['artifact', 'artifact', 'fake']) @@ -195,14 +170,15 @@ class StatsTest(tests.Test): def test_Commit(self): volume = db.Volume('local', model.RESOURCES) - volume['user'].create({'guid': 'user', 'name': 'user', 'color': '', 'pubkey': ''}) + volume['user'].create({'guid': 'user', 'name': 'user', 'pubkey': ''}) volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''}) volume['review'].create({'guid': 'review', 'context': 'context', 'title': '', 'content': '', 'rating': 5}) volume['feedback'].create({'guid': 'feedback', 'context': 'context', 'type': 'idea', 'title': '', 'content': ''}) volume['solution'].create({'guid': 'solution', 'context': 'context', 'feedback': 'feedback', 'content': ''}) volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''}) + volume['comment'].create({'guid': 'comment', 'context': 'context', 'message': ''}) - stats = Sniffer(volume) + stats = Sniffer(volume, 'stats/node') request = Request(method='GET', path=['user', 'user']) request.principal = 'user' stats.log(request) @@ -231,6 +207,7 @@ class StatsTest(tests.Test): ts = int(time.time()) stats.commit(ts) + stats.commit_objects() self.assertEqual(1, stats._stats['user'].total) self.assertEqual(1, stats._stats['context'].total) @@ -239,22 +216,22 @@ class StatsTest(tests.Test): self.assertEqual(1, stats._stats['solution'].total) self.assertEqual(1, stats._stats['artifact'].total) + print [[(j.name,) + i for i in j.get(j.last, j.last)] for j in Rrd('stats/node', 1)] + self.assertEqual([ + [('comment', ts, { + 'total': 1.0, + })], [('feedback', ts, { - 'solutions': 0.0, 'total': 1.0, - 'commented': 0.0, })], [('review', ts, { 'total': 1.0, - 'commented': 0.0, })], [('solution', ts, { 'total': 1.0, - 'commented': 0.0, })], [('artifact', ts, { - 'reviewed': 0.0, 'downloaded': 0.0, 'total': 1.0, })], @@ -263,7 +240,6 @@ class StatsTest(tests.Test): })], [('context', ts, { 'failed': 0.0, - 'reviewed': 0.0, 'downloaded': 0.0, 'total': 1.0, 'released': 0.0, @@ -282,7 +258,7 @@ class StatsTest(tests.Test): self.assertEqual([0, 0], volume['context'].get('context')['reviews']) self.assertEqual(0, volume['context'].get('context')['rating']) - stats = Sniffer(volume) + stats = Sniffer(volume, 'stats/node') request = Request(method='GET', path=['implementation', 'implementation', 'data']) request.principal = 'user' stats.log(request) @@ -295,18 +271,20 @@ class StatsTest(tests.Test): request.content = {'artifact': 'artifact', 'rating': 5} stats.log(request) stats.commit() + stats.commit_objects() self.assertEqual(1, volume['context'].get('context')['downloads']) self.assertEqual([1, 5], volume['context'].get('context')['reviews']) self.assertEqual(5, volume['context'].get('context')['rating']) stats.commit() + stats.commit_objects() self.assertEqual(1, volume['context'].get('context')['downloads']) self.assertEqual([1, 5], volume['context'].get('context')['reviews']) self.assertEqual(5, volume['context'].get('context')['rating']) - stats = Sniffer(volume) + stats = Sniffer(volume, 'stats/node') request = Request(method='GET', path=['implementation', 'implementation', 'data']) request.principal = 'user' stats.log(request) @@ -315,6 +293,7 @@ class StatsTest(tests.Test): request.content = {'context': 'context', 'rating': 1} stats.log(request) stats.commit() + stats.commit_objects() self.assertEqual(2, volume['context'].get('context')['downloads']) self.assertEqual([2, 6], volume['context'].get('context')['reviews']) @@ -330,7 +309,7 @@ class StatsTest(tests.Test): self.assertEqual([0, 0], volume['artifact'].get('artifact')['reviews']) self.assertEqual(0, volume['artifact'].get('artifact')['rating']) - stats = Sniffer(volume) + stats = Sniffer(volume, 'stats/node') request = Request(method='GET', path=['artifact', 'artifact', 'data']) request.principal = 'user' stats.log(request) @@ -339,12 +318,14 @@ class StatsTest(tests.Test): request.content = {'artifact': 'artifact', 'rating': 5} stats.log(request) stats.commit() + stats.commit_objects() self.assertEqual(1, volume['artifact'].get('artifact')['downloads']) self.assertEqual([1, 5], volume['artifact'].get('artifact')['reviews']) self.assertEqual(5, volume['artifact'].get('artifact')['rating']) stats.commit() + stats.commit_objects() self.assertEqual(1, volume['artifact'].get('artifact')['downloads']) self.assertEqual([1, 5], volume['artifact'].get('artifact')['reviews']) @@ -358,6 +339,7 @@ class StatsTest(tests.Test): request.content = {'artifact': 'artifact', 'rating': 1} stats.log(request) stats.commit() + stats.commit_objects() self.assertEqual(2, volume['artifact'].get('artifact')['downloads']) self.assertEqual([2, 6], volume['artifact'].get('artifact')['reviews']) diff --git a/tests/units/node/volume.py b/tests/units/node/volume.py index 40e052b..262b85d 100755 --- a/tests/units/node/volume.py +++ b/tests/units/node/volume.py @@ -351,7 +351,7 @@ class VolumeTest(tests.Test): def test_merge_UpdateStats(self): volume = db.Volume('db', model.RESOURCES) cp = NodeRoutes('guid', volume) - stats = Sniffer(volume) + stats = Sniffer(volume, 'stats/node') records = [ {'resource': 'context'}, @@ -445,28 +445,35 @@ class VolumeTest(tests.Test): 'stability': {'value': 'stable', 'mtime': 1.0}, 'notes': {'value': {}, 'mtime': 1.0}, }}, + {'resource': 'comment'}, + {'guid': 'comment', 'diff': { + 'guid': {'value': 'comment', 'mtime': 1.0}, + 'ctime': {'value': 1, 'mtime': 1.0}, + 'mtime': {'value': 1, 'mtime': 1.0}, + 'context': {'value': 'context', 'mtime': 1.0}, + 'message': {'value': {}, 'mtime': 1.0}, + }}, {'commit': [[1, 1]]}, ] merge(volume, records, stats=stats) ts = int(current_time()) stats.commit(ts) + stats.commit_objects() self.assertEqual([ + [('comment', ts, { + 'total': 1.0, + })], [('feedback', ts, { - 'solutions': 2.0, 'total': 2.0, - 'commented': 0.0, })], [('review', ts, { 'total': 2.0, - 'commented': 0.0, })], [('solution', ts, { 'total': 2.0, - 'commented': 0.0, })], [('artifact', ts, { - 'reviewed': 1.0, 'downloaded': 0.0, 'total': 1.0, })], @@ -475,7 +482,6 @@ class VolumeTest(tests.Test): })], [('context', ts, { 'failed': 0.0, - 'reviewed': 1.0, 'downloaded': 0.0, 'total': 1.0, 'released': 1.0, @@ -495,23 +501,22 @@ class VolumeTest(tests.Test): merge(volume, records, stats=stats) ts += 1 stats.commit(ts) + stats.commit_objects() self.assertEqual([ + [('comment', ts, { + 'total': 1.0, + })], [('feedback', ts, { - 'solutions': 1.0, 'total': 2.0, - 'commented': 0.0, })], [('review', ts, { 'total': 2.0, - 'commented': 0.0, })], [('solution', ts, { 'total': 2.0, - 'commented': 0.0, })], [('artifact', ts, { - 'reviewed': 0.0, 'downloaded': 0.0, 'total': 1.0, })], @@ -520,7 +525,6 @@ class VolumeTest(tests.Test): })], [('context', ts, { 'failed': 0.0, - 'reviewed': 0.0, 'downloaded': 0.0, 'total': 1.0, 'released': 0.0, @@ -549,23 +553,22 @@ class VolumeTest(tests.Test): merge(volume, records, stats=stats) ts += 1 stats.commit(ts) + stats.commit_objects() self.assertEqual([ + [('comment', ts, { + 'total': 1.0, + })], [('feedback', ts, { - 'solutions': 0.0, 'total': 0.0, - 'commented': 0.0, })], [('review', ts, { 'total': 0.0, - 'commented': 0.0, })], [('solution', ts, { 'total': 0.0, - 'commented': 0.0, })], [('artifact', ts, { - 'reviewed': 0.0, 'downloaded': 0.0, 'total': 0.0, })], @@ -574,7 +577,6 @@ class VolumeTest(tests.Test): })], [('context', ts, { 'failed': 0.0, - 'reviewed': 0.0, 'downloaded': 0.0, 'total': 0.0, 'released': 0.0, diff --git a/tests/units/toolkit/rrd.py b/tests/units/toolkit/rrd.py index b45b8b4..3434634 100755 --- a/tests/units/toolkit/rrd.py +++ b/tests/units/toolkit/rrd.py @@ -277,8 +277,8 @@ class RrdTest(tests.Test): for i in xrange((end_ts - start_ts) / 300): dbset.put({'f': i}, start_ts + i * 300) - prev_ts = 0 - prev_value = 0 + prev_ts = -1 + prev_value = -1 for ts, value in dbset.get(start_ts, end_ts, 86400): value = value['f'] assert ts > prev_ts -- cgit v0.9.1