Web   ·   Wiki   ·   Activities   ·   Blog   ·   Lists   ·   Chat   ·   Meeting   ·   Bugs   ·   Git   ·   Translate   ·   Archive   ·   People   ·   Donate
summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAleksey Lim <alsroot@sugarlabs.org>2013-11-21 20:26:30 (GMT)
committer Aleksey Lim <alsroot@sugarlabs.org>2013-11-21 20:26:30 (GMT)
commitebb59c1da33fddff54529860ad3b12aaa41ea4e8 (patch)
treeab3b583065df88ba967be1ccff181f0ea062c074
parent0e931f9bbb075f66f951057adec96afbf8b1c792 (diff)
Regenerate node stats
Diffstat
-rw-r--r--doc/objects.dia95
-rwxr-xr-xsugar-network-node9
-rw-r--r--sugar_network/client/routes.py1
-rw-r--r--sugar_network/db/index.py2
-rw-r--r--sugar_network/model/user.py4
-rw-r--r--sugar_network/node/routes.py126
-rw-r--r--sugar_network/node/stats_node.py221
-rw-r--r--sugar_network/toolkit/__init__.py22
-rw-r--r--sugar_network/toolkit/http.py3
-rw-r--r--sugar_network/toolkit/router.py2
-rw-r--r--sugar_network/toolkit/rrd.py79
-rw-r--r--tests/__init__.py1
-rwxr-xr-xtests/units/db/routes.py26
-rwxr-xr-xtests/units/node/node.py585
-rwxr-xr-xtests/units/node/stats_node.py102
-rwxr-xr-xtests/units/node/volume.py40
-rwxr-xr-xtests/units/toolkit/rrd.py4
17 files changed, 923 insertions, 399 deletions
diff --git a/doc/objects.dia b/doc/objects.dia
index 5ff9450..2c23de0 100644
--- a/doc/objects.dia
+++ b/doc/objects.dia
@@ -455,7 +455,7 @@
<dia:point val="3,6"/>
</dia:attribute>
<dia:attribute name="obj_bb">
- <dia:rectangle val="2.985,5.985;14.5475,28.815"/>
+ <dia:rectangle val="2.985,5.985;14.5475,22.015"/>
</dia:attribute>
<dia:attribute name="elem_corner">
<dia:point val="3,6"/>
@@ -464,7 +464,7 @@
<dia:real val="11.532500000000001"/>
</dia:attribute>
<dia:attribute name="elem_height">
- <dia:real val="22.800000000000004"/>
+ <dia:real val="16"/>
</dia:attribute>
<dia:attribute name="name">
<dia:string>#User#</dia:string>
@@ -556,29 +556,6 @@
<dia:attribute name="attributes">
<dia:composite type="umlattribute">
<dia:attribute name="name">
- <dia:string>#pubkey#</dia:string>
- </dia:attribute>
- <dia:attribute name="type">
- <dia:string>#str [WN]#</dia:string>
- </dia:attribute>
- <dia:attribute name="value">
- <dia:string>##</dia:string>
- </dia:attribute>
- <dia:attribute name="comment">
- <dia:string>#DSA public key for the key generated by Sugar Shell in user's profile#</dia:string>
- </dia:attribute>
- <dia:attribute name="visibility">
- <dia:enum val="0"/>
- </dia:attribute>
- <dia:attribute name="abstract">
- <dia:boolean val="false"/>
- </dia:attribute>
- <dia:attribute name="class_scope">
- <dia:boolean val="false"/>
- </dia:attribute>
- </dia:composite>
- <dia:composite type="umlattribute">
- <dia:attribute name="name">
<dia:string>#name#</dia:string>
</dia:attribute>
<dia:attribute name="type">
@@ -602,62 +579,16 @@
</dia:composite>
<dia:composite type="umlattribute">
<dia:attribute name="name">
- <dia:string>#color#</dia:string>
- </dia:attribute>
- <dia:attribute name="type">
- <dia:string>#str [R]#</dia:string>
- </dia:attribute>
- <dia:attribute name="value">
- <dia:string>##</dia:string>
- </dia:attribute>
- <dia:attribute name="comment">
- <dia:string>#Sugar colors pair in format "#RRGGBB,#RRGGBB"#</dia:string>
- </dia:attribute>
- <dia:attribute name="visibility">
- <dia:enum val="0"/>
- </dia:attribute>
- <dia:attribute name="abstract">
- <dia:boolean val="false"/>
- </dia:attribute>
- <dia:attribute name="class_scope">
- <dia:boolean val="false"/>
- </dia:attribute>
- </dia:composite>
- <dia:composite type="umlattribute">
- <dia:attribute name="name">
- <dia:string>#machine_sn#</dia:string>
- </dia:attribute>
- <dia:attribute name="type">
- <dia:string>#str [WA]#</dia:string>
- </dia:attribute>
- <dia:attribute name="value">
- <dia:string>#""#</dia:string>
- </dia:attribute>
- <dia:attribute name="comment">
- <dia:string>#XO serial number#</dia:string>
- </dia:attribute>
- <dia:attribute name="visibility">
- <dia:enum val="0"/>
- </dia:attribute>
- <dia:attribute name="abstract">
- <dia:boolean val="false"/>
- </dia:attribute>
- <dia:attribute name="class_scope">
- <dia:boolean val="false"/>
- </dia:attribute>
- </dia:composite>
- <dia:composite type="umlattribute">
- <dia:attribute name="name">
- <dia:string>#machine_uuid#</dia:string>
+ <dia:string>#location#</dia:string>
</dia:attribute>
<dia:attribute name="type">
- <dia:string>#str [WA]#</dia:string>
+ <dia:string>#str [R WA F]#</dia:string>
</dia:attribute>
<dia:attribute name="value">
<dia:string>#""#</dia:string>
</dia:attribute>
<dia:attribute name="comment">
- <dia:string>#XO UUID#</dia:string>
+ <dia:string>#User's real address in arbitrary form#</dia:string>
</dia:attribute>
<dia:attribute name="visibility">
<dia:enum val="0"/>
@@ -671,16 +602,16 @@
</dia:composite>
<dia:composite type="umlattribute">
<dia:attribute name="name">
- <dia:string>#location#</dia:string>
+ <dia:string>#birthday#</dia:string>
</dia:attribute>
<dia:attribute name="type">
- <dia:string>#str [R WA F]#</dia:string>
+ <dia:string>#int [R WA S]#</dia:string>
</dia:attribute>
<dia:attribute name="value">
- <dia:string>#""#</dia:string>
+ <dia:string>#0#</dia:string>
</dia:attribute>
<dia:attribute name="comment">
- <dia:string>#User's real address in arbitrary form#</dia:string>
+ <dia:string># User's birthday in seconds from UNIX epoch#</dia:string>
</dia:attribute>
<dia:attribute name="visibility">
<dia:enum val="0"/>
@@ -694,16 +625,16 @@
</dia:composite>
<dia:composite type="umlattribute">
<dia:attribute name="name">
- <dia:string>#birthday#</dia:string>
+ <dia:string>#pubkey#</dia:string>
</dia:attribute>
<dia:attribute name="type">
- <dia:string>#int [R WA S]#</dia:string>
+ <dia:string>#str [WN]#</dia:string>
</dia:attribute>
<dia:attribute name="value">
- <dia:string>#0#</dia:string>
+ <dia:string>##</dia:string>
</dia:attribute>
<dia:attribute name="comment">
- <dia:string>#User's birthday in seconds from UNIX epoch#</dia:string>
+ <dia:string>#DSA public key for the key generated by Sugar Shell in user's profile#</dia:string>
</dia:attribute>
<dia:attribute name="visibility">
<dia:enum val="0"/>
diff --git a/sugar-network-node b/sugar-network-node
index cd6961a..07bd96d 100755
--- a/sugar-network-node
+++ b/sugar-network-node
@@ -25,6 +25,7 @@ from gevent import monkey
import sugar_network_webui as webui
from sugar_network import db, node, client, toolkit
from sugar_network.node import stats_node, stats_user, obs
+from sugar_network.node.routes import generate_node_stats
from sugar_network.node.master import MasterRoutes
from sugar_network.node.slave import SlaveRoutes
from sugar_network.model import RESOURCES
@@ -118,6 +119,14 @@ class Application(application.Daemon):
path = self.args.pop(0)
self._ensure_instance().post(cmd='offline-sync', path=path)
+ @application.command(
+ 're-generate node statistics', name='restat')
+ def restat(self):
+ enforce(not self.check_for_instance(), 'Shutdown the server at first')
+ volume = db.Volume(node.data_root.value, RESOURCES)
+ volume.populate()
+ generate_node_stats(volume, join(node.stats_root.value, 'node'))
+
def _ensure_instance(self):
enforce(self.check_for_instance(), 'Node is not started')
return Connection('http://localhost:%s' %
diff --git a/sugar_network/client/routes.py b/sugar_network/client/routes.py
index b4680c0..e43eea5 100644
--- a/sugar_network/client/routes.py
+++ b/sugar_network/client/routes.py
@@ -552,7 +552,6 @@ class _Auth(http.SugarAuth):
import gconf
conf = gconf.client_get_default()
self._profile['name'] = conf.get_string('/desktop/sugar/user/nick')
- self._profile['color'] = conf.get_string('/desktop/sugar/user/color')
return http.SugarAuth.profile(self)
def __call__(self, nonce):
diff --git a/sugar_network/db/index.py b/sugar_network/db/index.py
index 4de1382..ab40527 100644
--- a/sugar_network/db/index.py
+++ b/sugar_network/db/index.py
@@ -170,7 +170,7 @@ class IndexReader(object):
enquire = self._enquire(request, query, order_by, group_by)
mset = self._call_db(enquire.get_mset, offset, limit, check_at_least)
- _logger.debug('Found in %s: %s time=%s total=%s parsed=%s',
+ _logger.debug('Found in %s: query=%r time=%s total=%s parsed=%s',
self.metadata.name, query, time.time() - start_timestamp,
mset.get_matches_estimated(), enquire.get_query())
diff --git a/sugar_network/model/user.py b/sugar_network/model/user.py
index e35cce8..69d0d42 100644
--- a/sugar_network/model/user.py
+++ b/sugar_network/model/user.py
@@ -23,10 +23,6 @@ class User(db.Resource):
def name(self, value):
return value
- @db.stored_property()
- def color(self, value):
- return value
-
@db.indexed_property(prefix='P', full_text=True, default='')
def location(self, value):
return value
diff --git a/sugar_network/node/routes.py b/sugar_network/node/routes.py
index 1182f28..f7fb9f4 100644
--- a/sugar_network/node/routes.py
+++ b/sugar_network/node/routes.py
@@ -55,12 +55,14 @@ class NodeRoutes(model.VolumeRoutes, model.FrontRoutes):
self._auth_config_mtime = 0
if stats_node.stats_node.value:
- self._stats = stats_node.Sniffer(volume)
+ stats_path = join(node.stats_root.value, 'node')
+ self._stats = stats_node.Sniffer(volume, stats_path)
coroutine.spawn(self._commit_stats)
def close(self):
if self._stats is not None:
self._stats.commit()
+ self._stats.commit_objects()
@property
def guid(self):
@@ -402,6 +404,91 @@ class NodeRoutes(model.VolumeRoutes, model.FrontRoutes):
return result
+def generate_node_stats(volume, path):
+ tmp_path = toolkit.mkdtemp()
+ new_stats = stats_node.Sniffer(volume, tmp_path, True)
+ old_stats = stats_node.Sniffer(volume, path)
+
+ def timeline(ts):
+ ts = long(ts)
+ end = long(time.time())
+ step = None
+
+ archives = {}
+ for rra in stats_node.stats_node_rras.value:
+ a_step, a_size = [long(i) for i in rra.split(':')[-2:]]
+ a_step *= stats_node.stats_node_step.value
+ a_start = end - min(end, a_step * a_size)
+ if archives.setdefault(a_start, a_step) > a_step:
+ archives[a_start] = a_step
+ archives = list(sorted(archives.items()))
+
+ try:
+ while ts <= end:
+ while not step or archives and ts >= archives[0][0]:
+ archive_start, step = archives.pop(0)
+ ts = max(ts / step * step, archive_start)
+ yield ts, ts + step - 1, step
+ ts += step
+ except GeneratorExit:
+ shutil.rmtree(tmp_path, ignore_errors=True)
+
+ start = next(volume['context'].find(limit=1, order_by='ctime')[0])['ctime']
+ for left, right, step in timeline(start):
+ for resource, props in [
+ ('user', []),
+ ('context', []),
+ ('implementation', ['context']),
+ ('artifact', ['context']),
+ ('feedback', ['context']),
+ ('solution', ['context', 'feedback']),
+ ('review', ['context', 'artifact', 'rating']),
+ ('report', ['context', 'implementation']),
+ ('comment', ['context', 'review', 'feedback', 'solution']),
+ ]:
+ objs, __ = volume[resource].find(
+ query='ctime:%s..%s' % (left, right))
+ for obj in objs:
+ request = Request(method='POST', path=[resource],
+ content=obj.properties(props))
+ new_stats.log(request)
+ for resource, props in [
+ ('user', ['layer']),
+ ('context', ['layer']),
+ ('implementation', ['layer']),
+ ('artifact', ['layer']),
+ ('feedback', ['layer', 'solution']),
+ ('solution', ['layer']),
+ ('review', ['layer']),
+ ('report', ['layer']),
+ ('comment', ['layer']),
+ ]:
+ objs, __ = volume[resource].find(
+ query='mtime:%s..%s' % (left, right))
+ for obj in objs:
+ if 'deleted' in obj['layer']:
+ request = Request(method='DELETE',
+ path=[resource, obj.guid])
+ else:
+ request = Request(method='PUT', path=[resource, obj.guid],
+ content=obj.properties(props))
+ new_stats.log(request)
+ downloaded = {}
+ for resource in ('context', 'artifact'):
+ stats = old_stats.report(
+ {resource: ['downloaded']}, left - step, right, 1)
+ if not stats.get(resource):
+ continue
+ stats = stats[resource][-1][1].get('downloaded')
+ if stats:
+ downloaded[resource] = {'downloaded': stats}
+ new_stats.commit(left + (right - left) / 2, downloaded)
+
+ new_stats.commit_objects(True)
+ shutil.rmtree(path)
+ shutil.move(tmp_path, path)
+
+
@contextmanager
def load_bundle(volume, request, bundle_path):
impl = request.copy()
@@ -488,6 +575,14 @@ def load_bundle(volume, request, bundle_path):
def _load_context_metadata(bundle, spec):
+
+ def convert(data, w, h):
+ result = toolkit.svg_to_png(data.getvalue(), w, h)
+ return {'blob': result,
+ 'mime_type': 'image/png',
+ 'digest': hashlib.sha1(result.getvalue()).hexdigest(),
+ }
+
result = {}
for prop in ('homepage', 'mime_types'):
if spec[prop]:
@@ -503,8 +598,8 @@ def _load_context_metadata(bundle, spec):
'mime_type': 'image/svg+xml',
'digest': hashlib.sha1(icon.getvalue()).hexdigest(),
},
- 'preview': _svg_to_png(icon.getvalue(), 160, 120),
- 'icon': _svg_to_png(icon.getvalue(), 55, 55),
+ 'preview': convert(icon, 160, 120),
+ 'icon': convert(icon, 55, 55),
})
except Exception:
exception(_logger, 'Failed to load icon')
@@ -538,28 +633,3 @@ def _load_context_metadata(bundle, spec):
exception(_logger, 'Gettext failed to read %r', mo_path[-1])
return result
-
-
-def _svg_to_png(data, w, h):
- import rsvg
- import cairo
-
- svg = rsvg.Handle(data=data)
- surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, w, h)
- context = cairo.Context(surface)
-
- scale = min(float(w) / svg.props.width, float(h) / svg.props.height)
- context.translate(
- int(w - svg.props.width * scale) / 2,
- int(h - svg.props.height * scale) / 2)
- context.scale(scale, scale)
- svg.render_cairo(context)
-
- result = StringIO()
- surface.write_to_png(result)
- result.seek(0)
-
- return {'blob': result,
- 'mime_type': 'image/png',
- 'digest': hashlib.sha1(result.getvalue()).hexdigest(),
- }
diff --git a/sugar_network/node/stats_node.py b/sugar_network/node/stats_node.py
index 61a4a28..fb9ef5d 100644
--- a/sugar_network/node/stats_node.py
+++ b/sugar_network/node/stats_node.py
@@ -14,9 +14,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import logging
-from os.path import join
-from sugar_network import node
from sugar_network.toolkit.rrd import Rrd
from sugar_network.toolkit import Option
@@ -32,11 +30,11 @@ stats_node_step = Option(
stats_node_rras = Option(
'comma separated list of RRAs for node RRD databases',
default=[
- 'RRA:AVERAGE:0.5:1:288', # one day with 5min step
- 'RRA:AVERAGE:0.5:3:672', # one week with 15min step
- 'RRA:AVERAGE:0.5:12:744', # one month with 1h step
- 'RRA:AVERAGE:0.5:144:732', # one year with 12h step
- 'RRA:AVERAGE:0.5:288:36600', # hundred years with 24h step
+ 'RRA:AVERAGE:0.5:1:864', # 3d with 5min step
+ 'RRA:AVERAGE:0.5:288:3660', # 10y with 1d step
+ 'RRA:AVERAGE:0.5:2880:366', # 10y with 10d step
+ 'RRA:AVERAGE:0.5:8640:122', # 10y with 30d step
+ 'RRA:AVERAGE:0.5:105408:10', # 10y with 1y step
],
type_cast=Option.list_cast, type_repr=Option.list_repr)
@@ -45,8 +43,7 @@ _logger = logging.getLogger('node.stats_node')
class Sniffer(object):
- def __init__(self, volume):
- path = join(node.stats_root.value, 'node')
+ def __init__(self, volume, path, reset=False):
_logger.info('Collect node stats in %r', path)
self._volume = volume
@@ -54,21 +51,76 @@ class Sniffer(object):
self._stats = {}
for name, cls in _STATS.items():
- self._stats[name] = cls(self._stats, volume)
+ stats = self._stats[name] = cls(self._stats, volume, reset)
+ fields = {}
+ for attr in dir(stats):
+ if attr[0] == '_' or attr[0].isupper() or \
+ type(getattr(stats, attr)) not in (int, long):
+ continue
+ if attr == 'total':
+ dst = 'GAUGE'
+ limit = 60 * 60 * 24 * 365
+ else:
+ dst = 'ABSOLUTE'
+ limit = stats_node_step.value
+ fields[attr] = 'DS:%s:%s:%s:U:U' % (attr, dst, limit)
+ if fields:
+ self._rrd[name].fields = fields
+
+ def __getitem__(self, name):
+ return self._rrd[name]
def log(self, request):
if request.cmd or request.resource not in _STATS:
return
self._stats[request.resource].log(request)
- def commit(self, timestamp=None):
+ def commit(self, timestamp=None, extra_values=None):
_logger.trace('Commit node stats')
for resource, stats in self._stats.items():
- values = stats.commit()
- if values is not None:
+ if resource not in self._rrd:
+ continue
+ values = {}
+ for field in self._rrd[resource].fields:
+ values[field] = getattr(stats, field)
+ if field != 'total':
+ setattr(stats, field, 0)
+ if extra_values and resource in extra_values:
+ values.update(extra_values[resource])
+ if values:
self._rrd[resource].put(values, timestamp=timestamp)
+ def commit_objects(self, reset=False):
+ _logger.trace('Commit object stats')
+
+ for resource, stats in self._stats.items():
+ obj = {
+ 'downloads': 0,
+ 'reviews': (0, 0),
+ }
+ directory = self._volume[resource]
+ for guid, obj_stats in stats.active.items():
+ if not obj_stats.reviews and not obj_stats.downloads:
+ continue
+ if not directory.exists(guid):
+ _logger.warning('Ignore stats for missed %r %s',
+ guid, resource)
+ continue
+ if not reset:
+ obj = directory.get(guid)
+ patch = {}
+ if obj_stats.downloads:
+ patch['downloads'] = obj_stats.downloads + obj['downloads']
+ if obj_stats.reviews:
+ reviews, rating = obj['reviews']
+ reviews += obj_stats.reviews
+ rating += obj_stats.rating
+ patch['reviews'] = [reviews, rating]
+ patch['rating'] = int(round(float(rating) / reviews))
+ directory.update(guid, patch)
+ stats.active.clear()
+
def report(self, dbs, start, end, records):
result = {}
@@ -77,11 +129,14 @@ class Sniffer(object):
return result
if not start:
- start = min([i.first for i in rdbs])
+ start = min([i.first for i in rdbs]) or 0
if not end:
- end = max([i.last for i in rdbs])
+ end = max([i.last for i in rdbs]) or 0
resolution = max(1, (end - start) / records)
+ _logger.debug('Report start=%s end=%s resolution=%s dbs=%r',
+ start, end, resolution, dbs)
+
for rdb in rdbs:
info = result[rdb.name] = []
for ts, ds_values in rdb.get(start, end, resolution):
@@ -105,15 +160,18 @@ class _Stats(object):
RESOURCE = None
OWNERS = []
- def __init__(self, stats, volume):
+ def __init__(self, stats, volume, reset):
+ self.active = {}
self._stats = stats
self._volume = volume
- self._directory = volume[self.RESOURCE]
- def log(self, request):
- pass
+ def __getitem__(self, guid):
+ result = self.active.get(guid)
+ if result is None:
+ result = self.active[guid] = _ObjectStats()
+ return result
- def commit(self):
+ def log(self, request):
pass
@@ -121,16 +179,10 @@ class _ResourceStats(_Stats):
total = 0
- def __init__(self, stats, volume):
- _Stats.__init__(self, stats, volume)
- self.total = volume[self.RESOURCE].find(limit=0)[1]
- self._active = {}
-
- def __getitem__(self, guid):
- result = self._active.get(guid)
- if result is None:
- result = self._active[guid] = _ObjectStats()
- return result
+ def __init__(self, stats, volume, reset):
+ _Stats.__init__(self, stats, volume, reset)
+ if not reset:
+ self.total = volume[self.RESOURCE].find(limit=0)[1]
def log(self, request):
if request.method == 'POST':
@@ -138,37 +190,9 @@ class _ResourceStats(_Stats):
elif request.method == 'DELETE':
self.total -= 1
- def commit(self):
- for guid, stats in self._active.items():
- if not stats.reviews and not stats.downloads:
- continue
- doc = self._directory.get(guid)
- updates = {}
- if stats.downloads:
- updates['downloads'] = stats.downloads + doc['downloads']
- if stats.reviews:
- reviews, rating = doc['reviews']
- reviews += stats.reviews
- rating += stats.rating
- updates['reviews'] = [reviews, rating]
- updates['rating'] = int(round(float(rating) / reviews))
- self._directory.update(guid, updates)
- self._active.clear()
-
- result = {}
- for attr in dir(self):
- if attr[0] == '_' or attr[0].isupper():
- continue
- value = getattr(self, attr)
- if type(value) in (set, dict):
- value = len(value)
- if type(value) in (int, long):
- result[attr] = value
-
- return result
-
def parse_context(self, request):
context = None
+ directory = self._volume[self.RESOURCE]
def parse_context(props):
for owner in self.OWNERS:
@@ -186,14 +210,14 @@ class _ResourceStats(_Stats):
elif self.RESOURCE == 'context':
context = request.guid
elif self.RESOURCE != 'user':
- context = self._directory.get(request.guid)['context']
+ context = directory.get(request.guid)['context']
elif request.method == 'PUT':
if self.RESOURCE == 'context':
context = request.guid
else:
context = request.content.get('context')
if not context:
- context = self._directory.get(request.guid)['context']
+ context = directory.get(request.guid)['context']
elif request.method == 'POST':
context = parse_context(request.content)
@@ -211,17 +235,8 @@ class _ContextStats(_ResourceStats):
released = 0
failed = 0
- reviewed = 0
downloaded = 0
- def commit(self):
- result = _ResourceStats.commit(self)
- self.released = 0
- self.failed = 0
- self.reviewed = 0
- self.downloaded = 0
- return result
-
class _ImplementationStats(_Stats):
@@ -231,7 +246,7 @@ class _ImplementationStats(_Stats):
def log(self, request):
if request.method == 'GET':
if request.prop == 'data':
- context = self._directory.get(request.guid)
+ context = self._volume[self.RESOURCE].get(request.guid)
self._stats['context'][context.context].downloads += 1
self._stats['context'].downloaded += 1
elif request.method == 'POST':
@@ -253,8 +268,6 @@ class _ReviewStats(_ResourceStats):
RESOURCE = 'review'
OWNERS = ['artifact', 'context']
- commented = 0
-
def log(self, request):
_ResourceStats.log(self, request)
@@ -262,17 +275,10 @@ class _ReviewStats(_ResourceStats):
if request.content.get('artifact'):
artifact = self._stats['artifact']
stats = artifact[request.content['artifact']]
- artifact.reviewed += 1
else:
stats = self._stats['context'][self.parse_context(request)]
- self._stats['context'].reviewed += 1
stats.reviews += 1
- stats.rating += request.content['rating']
-
- def commit(self):
- result = _ResourceStats.commit(self)
- self.commented = 0
- return result
+ stats.rating += request.content.get('rating') or 0
class _FeedbackStats(_ResourceStats):
@@ -280,56 +286,18 @@ class _FeedbackStats(_ResourceStats):
RESOURCE = 'feedback'
OWNERS = ['context']
- solutions = 0
- commented = 0
-
- def __init__(self, stats, volume):
- _ResourceStats.__init__(self, stats, volume)
- not_solved = volume['feedback'].find(limit=0, solution='')[1]
- self.solutions = self.total - not_solved
-
- def log(self, request):
- _ResourceStats.log(self, request)
-
- if request.method == 'POST':
- if request.content.get('solution'):
- self.solutions += 1
- elif request.method == 'PUT':
- if cmp(bool(self._directory.get(request.guid)['solution']),
- bool(request.content.get('solution'))):
- if request.content.get('solution'):
- self.solutions += 1
- else:
- self.solutions -= 1
- elif request.method == 'DELETE':
- if self._directory.get(request.guid)['solution']:
- self.solutions -= 1
-
- def commit(self):
- result = _ResourceStats.commit(self)
- self.commented = 0
- return result
-
class _SolutionStats(_ResourceStats):
RESOURCE = 'solution'
OWNERS = ['feedback']
- commented = 0
-
- def commit(self):
- result = _ResourceStats.commit(self)
- self.commented = 0
- return result
-
class _ArtifactStats(_ResourceStats):
RESOURCE = 'artifact'
OWNERS = ['context']
- reviewed = 0
downloaded = 0
def log(self, request):
@@ -340,25 +308,12 @@ class _ArtifactStats(_ResourceStats):
self[request.guid].downloads += 1
self.downloaded += 1
- def commit(self):
- result = _ResourceStats.commit(self)
- self.reviewed = 0
- self.downloaded = 0
- return result
-
-class _CommentStats(_Stats):
+class _CommentStats(_ResourceStats):
RESOURCE = 'comment'
OWNERS = ['solution', 'feedback', 'review']
- def log(self, request):
- if request.method == 'POST':
- for owner in ('solution', 'feedback', 'review'):
- if request.content.get(owner):
- self._stats[owner].commented += 1
- break
-
_STATS = {_UserStats.RESOURCE: _UserStats,
_ContextStats.RESOURCE: _ContextStats,
diff --git a/sugar_network/toolkit/__init__.py b/sugar_network/toolkit/__init__.py
index d56ec59..cbe1b91 100644
--- a/sugar_network/toolkit/__init__.py
+++ b/sugar_network/toolkit/__init__.py
@@ -21,6 +21,7 @@ import shutil
import logging
import tempfile
import collections
+from cStringIO import StringIO
from os.path import exists, join, islink, isdir, dirname, basename, abspath
from os.path import lexists, isfile
@@ -498,6 +499,27 @@ class mkdtemp(str):
shutil.rmtree(self)
+def svg_to_png(data, w, h):
+ import rsvg
+ import cairo
+
+ svg = rsvg.Handle(data=data)
+ surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, w, h)
+ context = cairo.Context(surface)
+
+ scale = min(float(w) / svg.props.width, float(h) / svg.props.height)
+ context.translate(
+ int(w - svg.props.width * scale) / 2,
+ int(h - svg.props.height * scale) / 2)
+ context.scale(scale, scale)
+ svg.render_cairo(context)
+
+ result = StringIO()
+ surface.write_to_png(result)
+ result.seek(0)
+ return result
+
+
def TemporaryFile(*args, **kwargs):
if cachedir.value:
if not exists(cachedir.value):
diff --git a/sugar_network/toolkit/http.py b/sugar_network/toolkit/http.py
index 2506873..d1b2fe7 100644
--- a/sugar_network/toolkit/http.py
+++ b/sugar_network/toolkit/http.py
@@ -310,6 +310,7 @@ class Connection(object):
sys.path.insert(0, sys_path)
from requests import Session, exceptions
Connection._Session = Session
+ # pylint: disable-msg=W0601
global ConnectionError
ConnectionError = exceptions.ConnectionError
@@ -341,7 +342,7 @@ class SugarAuth(object):
def __init__(self, key_path, profile=None):
self._key_path = abspath(expanduser(key_path))
- self._profile = profile or {'color': '#000000,#000000'}
+ self._profile = profile or {}
self._key = None
self._pubkey = None
self._login = None
diff --git a/sugar_network/toolkit/router.py b/sugar_network/toolkit/router.py
index 885bca9..6309301 100644
--- a/sugar_network/toolkit/router.py
+++ b/sugar_network/toolkit/router.py
@@ -575,7 +575,7 @@ class Router(object):
result = None
_logger.trace('%s call: request=%s response=%r result=%r',
- self, request.environ, response, result)
+ self, request.environ, response, repr(result)[:256])
start_response(response.status, response.items())
if result_streamed:
diff --git a/sugar_network/toolkit/rrd.py b/sugar_network/toolkit/rrd.py
index a5f879d..da58aff 100644
--- a/sugar_network/toolkit/rrd.py
+++ b/sugar_network/toolkit/rrd.py
@@ -18,9 +18,10 @@
import re
import os
import time
+import json
import bisect
import logging
-from os.path import exists, join
+from os.path import exists, join, splitext
_DB_FILENAME_RE = re.compile('(.*?)(-[0-9]+){0,1}\\.rrd$')
@@ -28,7 +29,7 @@ _INFO_RE = re.compile('([^[]+)\\[([^]]+)\\]\\.(.*)$')
_FETCH_PAGE = 256
-_logger = logging.getLogger('sugar_stats')
+_logger = logging.getLogger('rrd')
_rrdtool = None
@@ -76,8 +77,8 @@ class Rrd(object):
def get(self, name):
db = self._dbsets.get(name)
if db is None:
- db = self._dbsets[name] = \
- _DbSet(self._root, name, self._step, self._rras)
+ db = _DbSet(self._root, name, self._step, self._rras)
+ self._dbsets[name] = db
return db
@@ -89,10 +90,22 @@ class _DbSet(object):
self._step = step
self._rras = rras
self._revisions = []
- self._field_names = []
+ self._fields = None
+ self._field_names = None
self.__db = None
@property
+ def fields(self):
+ return self._field_names
+
+ @fields.setter
+ def fields(self, fields):
+ self._field_names = fields.keys()
+ self._field_names.sort()
+ self._fields = [str(fields[i]) for i in self._field_names]
+ _logger.debug('Set %r fields for %r', self._fields, self.name)
+
+ @property
def first(self):
if self._revisions:
return self._revisions[0].first
@@ -110,9 +123,11 @@ class _DbSet(object):
return db
def put(self, values, timestamp=None):
- if not self._field_names:
- self._field_names = values.keys()
- self._field_names.sort()
+ if not self.fields:
+ _logger.debug('Parse fields from the first put')
+ self.fields = dict([
+ (i, 'DS:%s:GAUGE:%s:U:U' % (i, self._step * 2))
+ for i in values])
if not timestamp:
timestamp = int(time.time())
@@ -133,7 +148,7 @@ class _DbSet(object):
_logger.debug('Put %r to %s', value, db.path)
- db.put(':'.join(value))
+ db.put(':'.join(value), timestamp)
def get(self, start=None, end=None, resolution=None):
if not self._revisions:
@@ -154,11 +169,11 @@ class _DbSet(object):
break
start = start - start % self._step - self._step
- end = min(end, start + _FETCH_PAGE * resolution)
- end -= end % self._step + self._step
+ last = min(end, start + _FETCH_PAGE * resolution)
+ last -= last % self._step + self._step
for db in reversed(revisions):
- db_end = min(end, db.last - self._step)
+ db_end = min(last, db.last - self._step)
if start > db_end:
break
(row_start, start, row_step), __, rows = _rrdtool.fetch(
@@ -169,17 +184,16 @@ class _DbSet(object):
'--resolution', str(resolution))
for raw_row in rows:
row_start += row_step
+ if row_start > end:
+ break
row = {}
- accept = False
for i, value in enumerate(raw_row):
- row[db.field_names[i]] = value
- accept = accept or value is not None
- if accept:
- yield row_start, row
+ row[db.field_names[i]] = value or .0
+ yield row_start, row
start = db_end + 1
def _get_db(self, timestamp):
- if self.__db is None and self._field_names:
+ if self.__db is None and self._fields:
if self._revisions:
db = self._revisions[-1]
if db.last >= timestamp:
@@ -189,14 +203,13 @@ class _DbSet(object):
return None
if db.step != self._step or db.rras != self._rras or \
db.field_names != self._field_names:
- db = self._create_db(self._field_names, db.revision + 1,
- db.last)
+ db = self._create_db(db.revision + 1, db.last)
else:
- db = self._create_db(self._field_names, 0, timestamp)
+ db = self._create_db(0, timestamp)
self.__db = db
return self.__db
- def _create_db(self, field_names, revision, timestamp):
+ def _create_db(self, revision, timestamp):
filename = self.name
if revision:
filename += '-%s' % revision
@@ -205,15 +218,11 @@ class _DbSet(object):
_logger.debug('Create %s database in %s start=%s step=%s',
filename, self._root, timestamp, self._step)
- fields = []
- for name in field_names:
- fields.append(str('DS:%s:GAUGE:%s:U:U' % (name, self._step * 2)))
-
_rrdtool.create(
str(join(self._root, filename)),
'--start', str(timestamp - self._step),
'--step', str(self._step),
- *(fields + self._rras))
+ *(self._fields + self._rras))
return self.load(filename, revision)
@@ -222,6 +231,7 @@ class _Db(object):
def __init__(self, path, revision=0):
self.path = str(path)
+ self._meta_path = splitext(path)[0] + '.meta'
self.revision = revision
self.fields = []
self.field_names = []
@@ -229,6 +239,7 @@ class _Db(object):
info = _rrdtool.info(self.path)
self.step = info['step']
+ self.first = 0
self.last = info['last_update']
fields = {}
@@ -257,13 +268,17 @@ class _Db(object):
self.fields.append(props)
self.field_names.append(name)
- def put(self, value):
+ if exists(self._meta_path):
+ with file(self._meta_path) as f:
+ self.first = json.load(f).get('first')
+
+ def put(self, value, timestamp):
+ if not self.first:
+ with file(self._meta_path, 'w') as f:
+ json.dump({'first': timestamp}, f)
+ self.first = timestamp
_rrdtool.update(self.path, str(value))
self.last = _rrdtool.info(self.path)['last_update']
- @property
- def first(self):
- return _rrdtool.first(self.path)
-
def __cmp__(self, other):
return cmp(self.revision, other.revision)
diff --git a/tests/__init__.py b/tests/__init__.py
index 58203a9..559020f 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -130,7 +130,6 @@ class Test(unittest.TestCase):
if tmp_root is None:
self.override(_Auth, 'profile', lambda self: {
'name': 'test',
- 'color': '#000000,#000000',
'pubkey': PUBKEY,
})
diff --git a/tests/units/db/routes.py b/tests/units/db/routes.py
index 4dd0fea..95aeb4e 100755
--- a/tests/units/db/routes.py
+++ b/tests/units/db/routes.py
@@ -1180,7 +1180,7 @@ class RoutesTest(tests.Test):
{'user': {'role': 2, 'order': 0}},
self.volume['document'].get(guid)['author'])
- self.volume['user'].create({'guid': 'user', 'color': '', 'pubkey': '', 'name': 'User'})
+ self.volume['user'].create({'guid': 'user', 'pubkey': '', 'name': 'User'})
guid = self.call('POST', ['document'], content={}, principal='user')
self.assertEqual(
@@ -1203,9 +1203,9 @@ class RoutesTest(tests.Test):
self.volume = db.Volume('db', [User, Document])
- self.volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'UserName1'})
- self.volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User Name2'})
- self.volume['user'].create({'guid': 'user3', 'color': '', 'pubkey': '', 'name': 'User Name 3'})
+ self.volume['user'].create({'guid': 'user1', 'pubkey': '', 'name': 'UserName1'})
+ self.volume['user'].create({'guid': 'user2', 'pubkey': '', 'name': 'User Name2'})
+ self.volume['user'].create({'guid': 'user3', 'pubkey': '', 'name': 'User Name 3'})
guid1 = self.call('POST', ['document'], content={}, principal='user1')
guid2 = self.call('POST', ['document'], content={}, principal='user2')
@@ -1245,9 +1245,9 @@ class RoutesTest(tests.Test):
self.volume = db.Volume('db', [User, Document])
- self.volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'})
- self.volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User2'})
- self.volume['user'].create({'guid': 'user3', 'color': '', 'pubkey': '', 'name': 'User3'})
+ self.volume['user'].create({'guid': 'user1', 'pubkey': '', 'name': 'User1'})
+ self.volume['user'].create({'guid': 'user2', 'pubkey': '', 'name': 'User2'})
+ self.volume['user'].create({'guid': 'user3', 'pubkey': '', 'name': 'User3'})
guid = self.call('POST', ['document'], content={}, principal='user1')
self.call('PUT', ['document', guid], cmd='useradd', user='user2', role=0)
@@ -1326,7 +1326,7 @@ class RoutesTest(tests.Test):
pass
self.volume = db.Volume('db', [User, Document])
- self.volume['user'].create({'guid': 'user', 'color': '', 'pubkey': '', 'name': 'User'})
+ self.volume['user'].create({'guid': 'user', 'pubkey': '', 'name': 'User'})
guid1 = self.call('POST', ['document'], content={}, principal='user')
self.assertEqual({'user': {'name': 'User', 'role': 3, 'order': 0}}, self.volume['document'].get(guid1)['author'])
@@ -1350,8 +1350,8 @@ class RoutesTest(tests.Test):
self.volume = db.Volume('db', [User, Document])
- self.volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'})
- self.volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User2'})
+ self.volume['user'].create({'guid': 'user1', 'pubkey': '', 'name': 'User1'})
+ self.volume['user'].create({'guid': 'user2', 'pubkey': '', 'name': 'User2'})
guid = self.call('POST', ['document'], content={}, principal='user1')
self.assertEqual([
@@ -1418,7 +1418,7 @@ class RoutesTest(tests.Test):
self.volume = db.Volume('db', [User, Document])
- self.volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'})
+ self.volume['user'].create({'guid': 'user1', 'pubkey': '', 'name': 'User1'})
guid = self.call('POST', ['document'], content={}, principal='user1')
self.call('PUT', ['document', guid], cmd='useradd', user='User2', role=0)
@@ -1470,8 +1470,8 @@ class RoutesTest(tests.Test):
self.volume = db.Volume('db', [User, Document])
- self.volume['user'].create({'guid': 'user1', 'color': '', 'pubkey': '', 'name': 'User1'})
- self.volume['user'].create({'guid': 'user2', 'color': '', 'pubkey': '', 'name': 'User2'})
+ self.volume['user'].create({'guid': 'user1', 'pubkey': '', 'name': 'User1'})
+ self.volume['user'].create({'guid': 'user2', 'pubkey': '', 'name': 'User2'})
guid = self.call('POST', ['document'], content={}, principal='user1')
self.call('PUT', ['document', guid], cmd='useradd', user='user2')
self.call('PUT', ['document', guid], cmd='useradd', user='User3')
diff --git a/tests/units/node/node.py b/tests/units/node/node.py
index b5abe78..9d6ae6f 100755
--- a/tests/units/node/node.py
+++ b/tests/units/node/node.py
@@ -20,7 +20,7 @@ from sugar_network.client import Connection, keyfile
from sugar_network.toolkit import http, coroutine
from sugar_network.toolkit.rrd import Rrd
from sugar_network.node import stats_user, stats_node, obs
-from sugar_network.node.routes import NodeRoutes
+from sugar_network.node.routes import NodeRoutes, generate_node_stats
from sugar_network.node.master import MasterRoutes
from sugar_network.model.user import User
from sugar_network.model.context import Context
@@ -48,7 +48,6 @@ class NodeTest(tests.Test):
call(cp, method='POST', document='user', principal=tests.UID, content={
'name': 'user',
- 'color': '',
'pubkey': tests.PUBKEY,
})
@@ -129,6 +128,7 @@ class NodeTest(tests.Test):
self.assertEqual({
'user': [
+ (ts + 0, {'total': 0.0}),
(ts + 3, {'total': 2.0}),
(ts + 6, {'total': 5.0}),
(ts + 9, {'total': 8.0}),
@@ -165,7 +165,7 @@ class NodeTest(tests.Test):
def test_HandleDeletes(self):
volume = db.Volume('db', model.RESOURCES)
- volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
+ volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
cp = NodeRoutes('guid', volume)
guid = call(cp, method='POST', document='context', principal=tests.UID, content={
@@ -200,7 +200,7 @@ class NodeTest(tests.Test):
def test_SimulateDeleteEvents(self):
volume = db.Volume('db', model.RESOURCES)
- volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
+ volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
cp = NodeRoutes('guid', volume)
guid = call(cp, method='POST', document='context', principal=tests.UID, content={
@@ -226,7 +226,6 @@ class NodeTest(tests.Test):
guid = call(cp, method='POST', document='user', principal=tests.UID2, content={
'name': 'user',
- 'color': '',
'pubkey': tests.PUBKEY,
})
assert guid is None
@@ -234,7 +233,7 @@ class NodeTest(tests.Test):
def test_UnauthorizedCommands(self):
volume = db.Volume('db', model.RESOURCES)
- volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
+ volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
class Routes(NodeRoutes):
@@ -272,7 +271,7 @@ class NodeTest(tests.Test):
pass
volume = db.Volume('db', [User, Document])
- volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
+ volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
cp = Routes('guid', volume)
guid = call(cp, method='POST', document='document', principal=tests.UID, content={})
@@ -287,7 +286,6 @@ class NodeTest(tests.Test):
call(cp, method='POST', document='user', principal=tests.UID2, content={
'name': 'user1',
- 'color': '',
'pubkey': tests.PUBKEY,
})
self.assertEqual('user1', call(cp, method='GET', document='user', guid=tests.UID, prop='name'))
@@ -310,8 +308,8 @@ class NodeTest(tests.Test):
return 'ok'
volume = db.Volume('db', [User])
- volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
- volume['user'].create({'guid': tests.UID2, 'name': 'test', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY2)}})
+ volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
+ volume['user'].create({'guid': tests.UID2, 'name': 'test', 'pubkey': {'blob': StringIO(tests.PUBKEY2)}})
cp = Routes('guid', volume)
self.assertRaises(http.Forbidden, call, cp, method='PROBE')
@@ -327,8 +325,8 @@ class NodeTest(tests.Test):
return value
volume = db.Volume('db', [User, Document])
- volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
- volume['user'].create({'guid': tests.UID2, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY2)}})
+ volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
+ volume['user'].create({'guid': tests.UID2, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY2)}})
cp = NodeRoutes('guid', volume)
guid = call(cp, method='POST', document='document', principal=tests.UID, content={'prop': '1'})
@@ -348,8 +346,8 @@ class NodeTest(tests.Test):
return value
volume = db.Volume('db', [User, Document])
- volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
- volume['user'].create({'guid': tests.UID2, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY2)}})
+ volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
+ volume['user'].create({'guid': tests.UID2, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY2)}})
cp = NodeRoutes('guid', volume)
guid = call(cp, method='POST', document='document', principal=tests.UID, content={'prop': '1'})
@@ -369,7 +367,7 @@ class NodeTest(tests.Test):
pass
volume = db.Volume('db', [User])
- volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
+ volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
cp = Routes('guid', volume)
self.assertRaises(http.Forbidden, call, cp, 'PROBE', principal=tests.UID)
@@ -407,7 +405,7 @@ class NodeTest(tests.Test):
def test_SetUser(self):
volume = db.Volume('db', model.RESOURCES)
- volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
+ volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
cp = NodeRoutes('guid', volume)
guid = call(cp, method='POST', document='context', principal=tests.UID, content={
@@ -422,7 +420,7 @@ class NodeTest(tests.Test):
def test_find_MaxLimit(self):
volume = db.Volume('db', model.RESOURCES)
- volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
+ volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
cp = NodeRoutes('guid', volume)
call(cp, method='POST', document='context', principal=tests.UID, content={
@@ -453,7 +451,7 @@ class NodeTest(tests.Test):
def test_DeletedDocuments(self):
volume = db.Volume('db', model.RESOURCES)
- volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
+ volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
cp = NodeRoutes('guid', volume)
guid = call(cp, method='POST', document='context', principal=tests.UID, content={
@@ -474,7 +472,7 @@ class NodeTest(tests.Test):
def test_CreateGUID(self):
# TODO Temporal security hole, see TODO
volume = db.Volume('db', model.RESOURCES)
- volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
+ volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
cp = NodeRoutes('guid', volume)
call(cp, method='POST', document='context', principal=tests.UID, content={
'guid': 'foo',
@@ -489,7 +487,7 @@ class NodeTest(tests.Test):
def test_CreateMalformedGUID(self):
volume = db.Volume('db', model.RESOURCES)
- volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
+ volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
cp = MasterRoutes('guid', volume)
self.assertRaises(RuntimeError, call, cp, method='POST', document='context', principal=tests.UID, content={
@@ -502,7 +500,7 @@ class NodeTest(tests.Test):
def test_FailOnExistedGUID(self):
volume = db.Volume('db', model.RESOURCES)
- volume['user'].create({'guid': tests.UID, 'name': 'user', 'color': '', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
+ volume['user'].create({'guid': tests.UID, 'name': 'user', 'pubkey': {'blob': StringIO(tests.PUBKEY)}})
cp = MasterRoutes('guid', volume)
guid = call(cp, method='POST', document='context', principal=tests.UID, content={
@@ -935,6 +933,551 @@ class NodeTest(tests.Test):
self.assertEqual(sorted(['origin', 'deleted']), sorted(volume['implementation'].get(impl2)['layer']))
self.assertEqual([], volume['implementation'].get(impl3)['layer'])
+ def test_generate_node_stats_Posts(self):
+ node.stats_root.value = 'stats'
+ stats_node.stats_node.value = True
+ stats_node.stats_node_rras.value = ['RRA:AVERAGE:0.5:1:10', 'RRA:AVERAGE:0.5:10:10']
+ volume = db.Volume('db', model.RESOURCES)
+
+ ts = 1000000000
+
+ volume['user'].create({
+ 'guid': 'user_1',
+ 'ctime': ts + 1,
+ 'mtime': ts + 1,
+ 'layer': [],
+ 'name': '',
+ })
+ volume['context'].create({
+ 'guid': 'context_1',
+ 'ctime': ts + 1,
+ 'mtime': ts + 1,
+ 'layer': [],
+ 'type': 'activity',
+ 'title': '',
+ 'summary': '',
+ 'description': '',
+ })
+ volume['implementation'].create({
+ 'guid': 'impl_1',
+ 'ctime': ts + 2,
+ 'mtime': ts + 2,
+ 'layer': [],
+ 'context': 'context_1',
+ 'license': ['GPL-3'],
+ 'version': '1',
+ })
+ volume['artifact'].create({
+ 'guid': 'artifact_1',
+ 'ctime': ts + 3,
+ 'mtime': ts + 3,
+ 'layer': [],
+ 'context': 'context_1',
+ 'type': 'preview',
+ 'title': '',
+ 'description': '',
+ })
+ volume['feedback'].create({
+ 'guid': 'feedback_1',
+ 'ctime': ts + 4,
+ 'mtime': ts + 4,
+ 'layer': [],
+ 'context': 'context_1',
+ 'title': '',
+ 'content': '',
+ 'type': 'idea',
+ })
+ volume['solution'].create({
+ 'guid': 'solution_1',
+ 'ctime': ts + 5,
+ 'mtime': ts + 5,
+ 'layer': [],
+ 'context': 'context_1',
+ 'feedback': 'feedback_1',
+ 'content': '',
+ })
+ volume['review'].create({
+ 'guid': 'review_1',
+ 'ctime': ts + 6,
+ 'mtime': ts + 6,
+ 'layer': [],
+ 'context': 'context_1',
+ 'rating': 1,
+ 'title': '',
+ 'content': '',
+ })
+ volume['review'].create({
+ 'guid': 'review_2',
+ 'ctime': ts + 6,
+ 'mtime': ts + 6,
+ 'layer': [],
+ 'context': 'context_1',
+ 'artifact': 'artifact_1',
+ 'rating': 2,
+ 'title': '',
+ 'content': '',
+ })
+ volume['comment'].create({
+ 'guid': 'comment_1',
+ 'ctime': ts + 7,
+ 'mtime': ts + 7,
+ 'layer': [],
+ 'context': 'context_1',
+ 'review': 'review_1',
+ 'message': '',
+ })
+ volume['comment'].create({
+ 'guid': 'comment_2',
+ 'ctime': ts + 7,
+ 'mtime': ts + 7,
+ 'layer': [],
+ 'context': 'context_1',
+ 'feedback': 'feedback_1',
+ 'message': '',
+ })
+ volume['comment'].create({
+ 'guid': 'comment_3',
+ 'ctime': ts + 7,
+ 'mtime': ts + 7,
+ 'layer': [],
+ 'context': 'context_1',
+ 'solution': 'solution_1',
+ 'message': '',
+ })
+ volume['report'].create({
+ 'guid': 'report_1',
+ 'ctime': ts + 8,
+ 'mtime': ts + 8,
+ 'layer': [],
+ 'context': 'context_1',
+ 'implementation': 'impl_1',
+ 'error': '',
+ })
+
+ volume['user'].create({
+ 'guid': 'user_2',
+ 'ctime': ts + 4,
+ 'mtime': ts + 4,
+ 'layer': [],
+ 'name': '',
+ })
+ volume['context'].create({
+ 'guid': 'context_2',
+ 'ctime': ts + 4,
+ 'mtime': ts + 4,
+ 'layer': [],
+ 'type': 'activity',
+ 'title': '',
+ 'summary': '',
+ 'description': '',
+ })
+ volume['implementation'].create({
+ 'guid': 'impl_2',
+ 'ctime': ts + 4,
+ 'mtime': ts + 4,
+ 'layer': [],
+ 'context': 'context_2',
+ 'license': ['GPL-3'],
+ 'version': '1',
+ })
+ volume['implementation'].create({
+ 'guid': 'impl_3',
+ 'ctime': ts + 4,
+ 'mtime': ts + 4,
+ 'layer': [],
+ 'context': 'context_2',
+ 'license': ['GPL-3'],
+ 'version': '1',
+ })
+ volume['review'].create({
+ 'guid': 'review_3',
+ 'ctime': ts + 4,
+ 'mtime': ts + 4,
+ 'layer': [],
+ 'context': 'context_2',
+ 'rating': 3,
+ 'title': '',
+ 'content': '',
+ })
+ volume['review'].create({
+ 'guid': 'review_4',
+ 'ctime': ts + 4,
+ 'mtime': ts + 4,
+ 'layer': [],
+ 'context': 'context_2',
+ 'rating': 4,
+ 'title': '',
+ 'content': '',
+ })
+ volume['report'].create({
+ 'guid': 'report_2',
+ 'ctime': ts + 4,
+ 'mtime': ts + 4,
+ 'layer': [],
+ 'context': 'context_2',
+ 'implementation': 'impl_1',
+ 'error': '',
+ })
+ volume['report'].create({
+ 'guid': 'report_3',
+ 'ctime': ts + 4,
+ 'mtime': ts + 4,
+ 'layer': [],
+ 'context': 'context_2',
+ 'implementation': 'impl_1',
+ 'error': '',
+ })
+ volume['artifact'].create({
+ 'guid': 'artifact_2',
+ 'ctime': ts + 4,
+ 'mtime': ts + 4,
+ 'layer': [],
+ 'context': 'context_2',
+ 'type': 'preview',
+ 'title': '',
+ 'description': '',
+ })
+ volume['feedback'].create({
+ 'guid': 'feedback_2',
+ 'ctime': ts + 4,
+ 'mtime': ts + 4,
+ 'layer': [],
+ 'context': 'context_2',
+ 'title': '',
+ 'content': '',
+ 'type': 'idea',
+ })
+ volume['solution'].create({
+ 'guid': 'solution_2',
+ 'ctime': ts + 4,
+ 'mtime': ts + 4,
+ 'layer': [],
+ 'context': 'context_2',
+ 'feedback': 'feedback_2',
+ 'content': '',
+ })
+ volume['comment'].create({
+ 'guid': 'comment_4',
+ 'ctime': ts + 4,
+ 'mtime': ts + 4,
+ 'layer': [],
+ 'context': 'context_2',
+ 'review': 'review_3',
+ 'message': '',
+ })
+ volume['comment'].create({
+ 'guid': 'comment_5',
+ 'ctime': ts + 4,
+ 'mtime': ts + 4,
+ 'layer': [],
+ 'context': 'context_2',
+ 'review': 'review_4',
+ 'message': '',
+ })
+ volume['comment'].create({
+ 'guid': 'comment_6',
+ 'ctime': ts + 4,
+ 'mtime': ts + 4,
+ 'layer': [],
+ 'context': 'context_2',
+ 'feedback': 'feedback_2',
+ 'message': '',
+ })
+ volume['comment'].create({
+ 'guid': 'comment_7',
+ 'ctime': ts + 4,
+ 'mtime': ts + 4,
+ 'layer': [],
+ 'context': 'context_2',
+ 'solution': 'solution_2',
+ 'message': '',
+ })
+
+ self.override(time, 'time', lambda: ts + 9)
+ old_stats = stats_node.Sniffer(volume, 'stats/node')
+ old_stats.log(Request(method='GET', path=['implementation', 'impl_1', 'data']))
+ old_stats.log(Request(method='GET', path=['artifact', 'artifact_1', 'data']))
+ old_stats.commit(ts + 1)
+ old_stats.commit_objects()
+ old_stats.commit(ts + 2)
+ old_stats.commit(ts + 3)
+ old_stats.log(Request(method='GET', path=['implementation', 'impl_1', 'data']))
+ old_stats.log(Request(method='GET', path=['implementation', 'impl_2', 'data']))
+ old_stats.commit(ts + 4)
+ old_stats.commit_objects()
+ old_stats.commit(ts + 5)
+ old_stats.commit(ts + 6)
+ old_stats.log(Request(method='GET', path=['artifact', 'artifact_1', 'data']))
+ old_stats.log(Request(method='GET', path=['artifact', 'artifact_2', 'data']))
+ old_stats.commit(ts + 7)
+ old_stats.commit_objects()
+ old_stats.commit(ts + 8)
+ old_stats.commit_objects()
+
+ generate_node_stats(volume, 'stats/node')
+ cp = NodeRoutes('guid', volume)
+
+ self.assertEqual({
+ 'user': [
+ (ts + 1, {'total': 1.0}),
+ (ts + 2, {'total': 1.0}),
+ (ts + 3, {'total': 1.0}),
+ (ts + 4, {'total': 2.0}),
+ (ts + 5, {'total': 2.0}),
+ (ts + 6, {'total': 2.0}),
+ (ts + 7, {'total': 2.0}),
+ (ts + 8, {'total': 2.0}),
+ (ts + 9, {'total': 2.0}),
+ ],
+ 'context': [
+ (ts + 1, {'total': 1.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 1.0}),
+ (ts + 2, {'total': 1.0, 'released': 1.0, 'failed': 0.0, 'downloaded': 0.0}),
+ (ts + 3, {'total': 1.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}),
+ (ts + 4, {'total': 2.0, 'released': 2.0, 'failed': 2.0, 'downloaded': 2.0}),
+ (ts + 5, {'total': 2.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}),
+ (ts + 6, {'total': 2.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}),
+ (ts + 7, {'total': 2.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}),
+ (ts + 8, {'total': 2.0, 'released': 0.0, 'failed': 1.0, 'downloaded': 0.0}),
+ (ts + 9, {'total': 2.0, 'released': 0.0, 'failed': 0.0, 'downloaded': 0.0}),
+ ],
+ 'review': [
+ (ts + 1, {'total': 0.0}),
+ (ts + 2, {'total': 0.0}),
+ (ts + 3, {'total': 0.0}),
+ (ts + 4, {'total': 2.0}),
+ (ts + 5, {'total': 2.0}),
+ (ts + 6, {'total': 4.0}),
+ (ts + 7, {'total': 4.0}),
+ (ts + 8, {'total': 4.0}),
+ (ts + 9, {'total': 4.0}),
+ ],
+ 'feedback': [
+ (ts + 1, {'total': 0.0}),
+ (ts + 2, {'total': 0.0}),
+ (ts + 3, {'total': 0.0}),
+ (ts + 4, {'total': 2.0}),
+ (ts + 5, {'total': 2.0}),
+ (ts + 6, {'total': 2.0}),
+ (ts + 7, {'total': 2.0}),
+ (ts + 8, {'total': 2.0}),
+ (ts + 9, {'total': 2.0}),
+ ],
+ 'solution': [
+ (ts + 1, {'total': 0.0}),
+ (ts + 2, {'total': 0.0}),
+ (ts + 3, {'total': 0.0}),
+ (ts + 4, {'total': 1.0}),
+ (ts + 5, {'total': 2.0}),
+ (ts + 6, {'total': 2.0}),
+ (ts + 7, {'total': 2.0}),
+ (ts + 8, {'total': 2.0}),
+ (ts + 9, {'total': 2.0}),
+ ],
+ 'artifact': [
+ (ts + 1, {'total': 0.0, 'downloaded': 1.0}),
+ (ts + 2, {'total': 0.0, 'downloaded': 0.0}),
+ (ts + 3, {'total': 1.0, 'downloaded': 0.0}),
+ (ts + 4, {'total': 2.0, 'downloaded': 0.0}),
+ (ts + 5, {'total': 2.0, 'downloaded': 0.0}),
+ (ts + 6, {'total': 2.0, 'downloaded': 0.0}),
+ (ts + 7, {'total': 2.0, 'downloaded': 2.0}),
+ (ts + 8, {'total': 2.0, 'downloaded': 0.0}),
+ (ts + 9, {'total': 2.0, 'downloaded': 0.0}),
+ ],
+ 'comment': [
+ (ts + 1, {'total': 0.0}),
+ (ts + 2, {'total': 0.0}),
+ (ts + 3, {'total': 0.0}),
+ (ts + 4, {'total': 4.0}),
+ (ts + 5, {'total': 4.0}),
+ (ts + 6, {'total': 4.0}),
+ (ts + 7, {'total': 7.0}),
+ (ts + 8, {'total': 7.0}),
+ (ts + 9, {'total': 7.0}),
+ ],
+ },
+ call(cp, method='GET', cmd='stats', source=[
+ 'user.total',
+ 'context.total',
+ 'context.released',
+ 'context.failed',
+ 'context.downloaded',
+ 'review.total',
+ 'feedback.total',
+ 'solution.total',
+ 'artifact.total',
+ 'artifact.downloaded',
+ 'comment.total',
+ ], start=ts + 1, end=ts + 10))
+
+ self.assertEqual({
+ 'downloads': 2,
+ 'rating': 1,
+ 'reviews': [1, 1],
+ },
+ volume['context'].get('context_1').properties(['downloads', 'rating', 'reviews']))
+ self.assertEqual({
+ 'downloads': 1,
+ 'rating': 4,
+ 'reviews': [2, 7],
+ },
+ volume['context'].get('context_2').properties(['downloads', 'rating', 'reviews']))
+ self.assertEqual({
+ 'downloads': 2,
+ 'rating': 2,
+ 'reviews': [1, 2],
+ },
+ volume['artifact'].get('artifact_1').properties(['downloads', 'rating', 'reviews']))
+ self.assertEqual({
+ 'downloads': 1,
+ 'rating': 0,
+ 'reviews': [0, 0],
+ },
+ volume['artifact'].get('artifact_2').properties(['downloads', 'rating', 'reviews']))
+
+ def test_generate_node_stats_Deletes(self):
+ node.stats_root.value = 'stats'
+ stats_node.stats_node.value = True
+ stats_node.stats_node_rras.value = ['RRA:AVERAGE:0.5:1:10', 'RRA:AVERAGE:0.5:10:10']
+ volume = db.Volume('db', model.RESOURCES)
+
+ ts = 1000000000
+
+ volume['user'].create({
+ 'guid': 'user_1',
+ 'ctime': ts + 1,
+ 'mtime': ts + 2,
+ 'layer': ['deleted'],
+ 'name': '',
+ })
+ volume['context'].create({
+ 'guid': 'context_1',
+ 'ctime': ts + 1,
+ 'mtime': ts + 2,
+ 'layer': ['deleted'],
+ 'type': 'activity',
+ 'title': '',
+ 'summary': '',
+ 'description': '',
+ })
+ volume['implementation'].create({
+ 'guid': 'impl_1',
+ 'ctime': ts + 1,
+ 'mtime': ts + 2,
+ 'layer': ['deleted'],
+ 'context': 'context_1',
+ 'license': ['GPL-3'],
+ 'version': '1',
+ })
+ volume['artifact'].create({
+ 'guid': 'artifact_1',
+ 'ctime': ts + 1,
+ 'mtime': ts + 2,
+ 'layer': ['deleted'],
+ 'context': 'context_1',
+ 'type': 'preview',
+ 'title': '',
+ 'description': '',
+ })
+ volume['feedback'].create({
+ 'guid': 'feedback_1',
+ 'ctime': ts + 1,
+ 'mtime': ts + 2,
+ 'layer': ['deleted'],
+ 'context': 'context_1',
+ 'title': '',
+ 'content': '',
+ 'type': 'idea',
+ })
+ volume['solution'].create({
+ 'guid': 'solution_1',
+ 'ctime': ts + 1,
+ 'mtime': ts + 2,
+ 'layer': ['deleted'],
+ 'context': 'context_1',
+ 'feedback': 'feedback_1',
+ 'content': '',
+ })
+ volume['review'].create({
+ 'guid': 'review_1',
+ 'ctime': ts + 1,
+ 'mtime': ts + 2,
+ 'layer': ['deleted'],
+ 'context': 'context_1',
+ 'rating': 1,
+ 'title': '',
+ 'content': '',
+ })
+ volume['comment'].create({
+ 'guid': 'comment_1',
+ 'ctime': ts + 1,
+ 'mtime': ts + 2,
+ 'layer': ['deleted'],
+ 'context': 'context_1',
+ 'review': 'review_1',
+ 'message': '',
+ })
+ volume['report'].create({
+ 'guid': 'report_1',
+ 'ctime': ts + 1,
+ 'mtime': ts + 2,
+ 'layer': ['deleted'],
+ 'context': 'context_1',
+ 'implementation': 'impl_1',
+ 'error': '',
+ })
+
+ self.override(time, 'time', lambda: ts + 9)
+ generate_node_stats(volume, 'stats/node')
+ cp = NodeRoutes('guid', volume)
+
+ self.assertEqual({
+ 'user': [
+ (ts + 1, {'total': 1.0}),
+ (ts + 2, {'total': 0.0}),
+ (ts + 3, {'total': 0.0}),
+ ],
+ 'context': [
+ (ts + 1, {'total': 1.0}),
+ (ts + 2, {'total': 0.0}),
+ (ts + 3, {'total': 0.0}),
+ ],
+ 'review': [
+ (ts + 1, {'total': 1.0}),
+ (ts + 2, {'total': 0.0}),
+ (ts + 3, {'total': 0.0}),
+ ],
+ 'feedback': [
+ (ts + 1, {'total': 1.0}),
+ (ts + 2, {'total': 0.0}),
+ (ts + 3, {'total': 0.0}),
+ ],
+ 'solution': [
+ (ts + 1, {'total': 1.0}),
+ (ts + 2, {'total': 0.0}),
+ (ts + 3, {'total': 0.0}),
+ ],
+ 'artifact': [
+ (ts + 1, {'total': 1.0}),
+ (ts + 2, {'total': 0.0}),
+ (ts + 3, {'total': 0.0}),
+ ],
+ 'comment': [
+ (ts + 1, {'total': 1.0}),
+ (ts + 2, {'total': 0.0}),
+ (ts + 3, {'total': 0.0}),
+ ],
+ },
+ call(cp, method='GET', cmd='stats', source=[
+ 'user.total',
+ 'context.total',
+ 'review.total',
+ 'feedback.total',
+ 'solution.total',
+ 'artifact.total',
+ 'comment.total',
+ ], start=ts + 1, end=ts + 3))
+
def call(routes, method, document=None, guid=None, prop=None, principal=None, content=None, **kwargs):
path = ['']
diff --git a/tests/units/node/stats_node.py b/tests/units/node/stats_node.py
index a0ae5d0..da2f3e7 100755
--- a/tests/units/node/stats_node.py
+++ b/tests/units/node/stats_node.py
@@ -16,16 +16,15 @@ class StatsTest(tests.Test):
def test_InitializeTotals(self):
volume = db.Volume('local', model.RESOURCES)
- stats = Sniffer(volume)
+ stats = Sniffer(volume, 'stats/node')
self.assertEqual(0, stats._stats['user'].total)
self.assertEqual(0, stats._stats['context'].total)
self.assertEqual(0, stats._stats['review'].total)
self.assertEqual(0, stats._stats['feedback'].total)
- self.assertEqual(0, stats._stats['feedback'].solutions)
self.assertEqual(0, stats._stats['solution'].total)
self.assertEqual(0, stats._stats['artifact'].total)
- volume['user'].create({'guid': 'user', 'name': 'user', 'color': '', 'pubkey': ''})
+ volume['user'].create({'guid': 'user', 'name': 'user', 'pubkey': ''})
volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''})
volume['review'].create({'guid': 'review', 'context': 'context', 'title': '', 'content': '', 'rating': 5})
volume['feedback'].create({'guid': 'feedback', 'context': 'context', 'type': 'idea', 'title': '', 'content': ''})
@@ -33,18 +32,17 @@ class StatsTest(tests.Test):
volume['solution'].create({'guid': 'solution', 'context': 'context', 'feedback': 'feedback', 'content': ''})
volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''})
- stats = Sniffer(volume)
+ stats = Sniffer(volume, 'stats/node')
self.assertEqual(1, stats._stats['user'].total)
self.assertEqual(1, stats._stats['context'].total)
self.assertEqual(1, stats._stats['review'].total)
self.assertEqual(2, stats._stats['feedback'].total)
- self.assertEqual(1, stats._stats['feedback'].solutions)
self.assertEqual(1, stats._stats['solution'].total)
self.assertEqual(1, stats._stats['artifact'].total)
def test_POSTs(self):
volume = db.Volume('local', model.RESOURCES)
- stats = Sniffer(volume)
+ stats = Sniffer(volume, 'stats/node')
request = Request(method='POST', path=['context'])
request.principal = 'user'
@@ -55,7 +53,7 @@ class StatsTest(tests.Test):
def test_DELETEs(self):
volume = db.Volume('local', model.RESOURCES)
- stats = Sniffer(volume)
+ stats = Sniffer(volume, 'stats/node')
request = Request(method='DELETE', path=['context'])
request.principal = 'user'
@@ -64,33 +62,9 @@ class StatsTest(tests.Test):
stats.log(request)
self.assertEqual(-3, stats._stats['context'].total)
- def test_FeedbackSolutions(self):
- volume = db.Volume('local', model.RESOURCES)
- stats = Sniffer(volume)
- volume['feedback'].create({'guid': 'guid', 'context': 'context', 'type': 'idea', 'title': '', 'content': ''})
-
- request = Request(method='PUT', path=['feedback', 'guid'])
- request.principal = 'user'
- request.content = {}
- stats.log(request)
- self.assertEqual(0, stats._stats['feedback'].solutions)
-
- request.content = {'solution': 'solution'}
- stats.log(request)
- self.assertEqual(1, stats._stats['feedback'].solutions)
-
- request.content = {'solution': None}
- stats.log(request)
- self.assertEqual(1, stats._stats['feedback'].solutions)
-
- volume['feedback'].update('guid', {'solution': 'exists'})
- request.content = {'solution': None}
- stats.log(request)
- self.assertEqual(0, stats._stats['feedback'].solutions)
-
def test_Comments(self):
volume = db.Volume('local', model.RESOURCES)
- stats = Sniffer(volume)
+ stats = Sniffer(volume, 'stats/node')
volume['solution'].create({'guid': 'solution', 'context': 'context', 'feedback': 'feedback', 'content': ''})
volume['feedback'].create({'guid': 'feedback', 'context': 'context', 'type': 'idea', 'title': '', 'content': ''})
volume['review'].create({'guid': 'review', 'context': 'context', 'title': '', 'content': '', 'rating': 5})
@@ -99,50 +73,51 @@ class StatsTest(tests.Test):
request.principal = 'user'
request.content = {'solution': 'solution'}
stats.log(request)
- self.assertEqual(1, stats._stats['solution'].commented)
+ self.assertEqual(1, stats._stats['comment'].total)
request = Request(method='POST', path=['comment'])
request.principal = 'user'
request.content = {'feedback': 'feedback'}
stats.log(request)
- self.assertEqual(1, stats._stats['feedback'].commented)
+ self.assertEqual(2, stats._stats['comment'].total)
request = Request(method='POST', path=['comment'])
request.principal = 'user'
request.content = {'review': 'review'}
stats.log(request)
- self.assertEqual(1, stats._stats['review'].commented)
+ self.assertEqual(3, stats._stats['comment'].total)
def test_Reviewes(self):
volume = db.Volume('local', model.RESOURCES)
- stats = Sniffer(volume)
+ stats = Sniffer(volume, 'stats/node')
volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''})
volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''})
request = Request(method='POST', path=['review'])
request.principal = 'user'
- request.content = {'context': 'context', 'rating': 0}
+ request.content = {'context': 'context', 'rating': 1}
stats.log(request)
- self.assertEqual(1, stats._stats['context'].reviewed)
- self.assertEqual(0, stats._stats['artifact'].reviewed)
+ self.assertEqual(1, stats._stats['review'].total)
request = Request(method='POST', path=['review'])
request.principal = 'user'
- request.content = {'context': 'context', 'artifact': '', 'rating': 0}
+ request.content = {'context': 'context', 'artifact': '', 'rating': 2}
stats.log(request)
- self.assertEqual(2, stats._stats['context'].reviewed)
- self.assertEqual(0, stats._stats['artifact'].reviewed)
+ self.assertEqual(2, stats._stats['review'].total)
request = Request(method='POST', path=['review'])
request.principal = 'user'
- request.content = {'artifact': 'artifact', 'rating': 0}
+ request.content = {'artifact': 'artifact', 'rating': 3}
stats.log(request)
- self.assertEqual(2, stats._stats['context'].reviewed)
- self.assertEqual(1, stats._stats['artifact'].reviewed)
+ self.assertEqual(3, stats._stats['review'].total)
+
+ stats.commit_objects()
+ self.assertEqual([2, 3], volume['context'].get('context')['reviews'])
+ self.assertEqual([1, 3], volume['artifact'].get('artifact')['reviews'])
def test_ContextDownloaded(self):
volume = db.Volume('local', model.RESOURCES)
- stats = Sniffer(volume)
+ stats = Sniffer(volume, 'stats/node')
volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''})
volume['implementation'].create({'guid': 'implementation', 'context': 'context', 'license': 'GPLv3', 'version': '1', 'date': 0, 'stability': 'stable', 'notes': ''})
@@ -158,7 +133,7 @@ class StatsTest(tests.Test):
def test_ContextReleased(self):
volume = db.Volume('local', model.RESOURCES)
- stats = Sniffer(volume)
+ stats = Sniffer(volume, 'stats/node')
volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''})
request = Request(method='POST', path=['implementation'])
@@ -169,7 +144,7 @@ class StatsTest(tests.Test):
def test_ContextFailed(self):
volume = db.Volume('local', model.RESOURCES)
- stats = Sniffer(volume)
+ stats = Sniffer(volume, 'stats/node')
volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''})
request = Request(method='POST', path=['report'])
@@ -180,7 +155,7 @@ class StatsTest(tests.Test):
def test_ArtifactDownloaded(self):
volume = db.Volume('local', model.RESOURCES)
- stats = Sniffer(volume)
+ stats = Sniffer(volume, 'stats/node')
volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''})
request = Request(method='GET', path=['artifact', 'artifact', 'fake'])
@@ -195,14 +170,15 @@ class StatsTest(tests.Test):
def test_Commit(self):
volume = db.Volume('local', model.RESOURCES)
- volume['user'].create({'guid': 'user', 'name': 'user', 'color': '', 'pubkey': ''})
+ volume['user'].create({'guid': 'user', 'name': 'user', 'pubkey': ''})
volume['context'].create({'guid': 'context', 'type': 'activity', 'title': '', 'summary': '', 'description': ''})
volume['review'].create({'guid': 'review', 'context': 'context', 'title': '', 'content': '', 'rating': 5})
volume['feedback'].create({'guid': 'feedback', 'context': 'context', 'type': 'idea', 'title': '', 'content': ''})
volume['solution'].create({'guid': 'solution', 'context': 'context', 'feedback': 'feedback', 'content': ''})
volume['artifact'].create({'guid': 'artifact', 'type': 'instance', 'context': 'context', 'title': '', 'description': ''})
+ volume['comment'].create({'guid': 'comment', 'context': 'context', 'message': ''})
- stats = Sniffer(volume)
+ stats = Sniffer(volume, 'stats/node')
request = Request(method='GET', path=['user', 'user'])
request.principal = 'user'
stats.log(request)
@@ -231,6 +207,7 @@ class StatsTest(tests.Test):
ts = int(time.time())
stats.commit(ts)
+ stats.commit_objects()
self.assertEqual(1, stats._stats['user'].total)
self.assertEqual(1, stats._stats['context'].total)
@@ -239,22 +216,22 @@ class StatsTest(tests.Test):
self.assertEqual(1, stats._stats['solution'].total)
self.assertEqual(1, stats._stats['artifact'].total)
+ print [[(j.name,) + i for i in j.get(j.last, j.last)] for j in Rrd('stats/node', 1)]
+
self.assertEqual([
+ [('comment', ts, {
+ 'total': 1.0,
+ })],
[('feedback', ts, {
- 'solutions': 0.0,
'total': 1.0,
- 'commented': 0.0,
})],
[('review', ts, {
'total': 1.0,
- 'commented': 0.0,
})],
[('solution', ts, {
'total': 1.0,
- 'commented': 0.0,
})],
[('artifact', ts, {
- 'reviewed': 0.0,
'downloaded': 0.0,
'total': 1.0,
})],
@@ -263,7 +240,6 @@ class StatsTest(tests.Test):
})],
[('context', ts, {
'failed': 0.0,
- 'reviewed': 0.0,
'downloaded': 0.0,
'total': 1.0,
'released': 0.0,
@@ -282,7 +258,7 @@ class StatsTest(tests.Test):
self.assertEqual([0, 0], volume['context'].get('context')['reviews'])
self.assertEqual(0, volume['context'].get('context')['rating'])
- stats = Sniffer(volume)
+ stats = Sniffer(volume, 'stats/node')
request = Request(method='GET', path=['implementation', 'implementation', 'data'])
request.principal = 'user'
stats.log(request)
@@ -295,18 +271,20 @@ class StatsTest(tests.Test):
request.content = {'artifact': 'artifact', 'rating': 5}
stats.log(request)
stats.commit()
+ stats.commit_objects()
self.assertEqual(1, volume['context'].get('context')['downloads'])
self.assertEqual([1, 5], volume['context'].get('context')['reviews'])
self.assertEqual(5, volume['context'].get('context')['rating'])
stats.commit()
+ stats.commit_objects()
self.assertEqual(1, volume['context'].get('context')['downloads'])
self.assertEqual([1, 5], volume['context'].get('context')['reviews'])
self.assertEqual(5, volume['context'].get('context')['rating'])
- stats = Sniffer(volume)
+ stats = Sniffer(volume, 'stats/node')
request = Request(method='GET', path=['implementation', 'implementation', 'data'])
request.principal = 'user'
stats.log(request)
@@ -315,6 +293,7 @@ class StatsTest(tests.Test):
request.content = {'context': 'context', 'rating': 1}
stats.log(request)
stats.commit()
+ stats.commit_objects()
self.assertEqual(2, volume['context'].get('context')['downloads'])
self.assertEqual([2, 6], volume['context'].get('context')['reviews'])
@@ -330,7 +309,7 @@ class StatsTest(tests.Test):
self.assertEqual([0, 0], volume['artifact'].get('artifact')['reviews'])
self.assertEqual(0, volume['artifact'].get('artifact')['rating'])
- stats = Sniffer(volume)
+ stats = Sniffer(volume, 'stats/node')
request = Request(method='GET', path=['artifact', 'artifact', 'data'])
request.principal = 'user'
stats.log(request)
@@ -339,12 +318,14 @@ class StatsTest(tests.Test):
request.content = {'artifact': 'artifact', 'rating': 5}
stats.log(request)
stats.commit()
+ stats.commit_objects()
self.assertEqual(1, volume['artifact'].get('artifact')['downloads'])
self.assertEqual([1, 5], volume['artifact'].get('artifact')['reviews'])
self.assertEqual(5, volume['artifact'].get('artifact')['rating'])
stats.commit()
+ stats.commit_objects()
self.assertEqual(1, volume['artifact'].get('artifact')['downloads'])
self.assertEqual([1, 5], volume['artifact'].get('artifact')['reviews'])
@@ -358,6 +339,7 @@ class StatsTest(tests.Test):
request.content = {'artifact': 'artifact', 'rating': 1}
stats.log(request)
stats.commit()
+ stats.commit_objects()
self.assertEqual(2, volume['artifact'].get('artifact')['downloads'])
self.assertEqual([2, 6], volume['artifact'].get('artifact')['reviews'])
diff --git a/tests/units/node/volume.py b/tests/units/node/volume.py
index 40e052b..262b85d 100755
--- a/tests/units/node/volume.py
+++ b/tests/units/node/volume.py
@@ -351,7 +351,7 @@ class VolumeTest(tests.Test):
def test_merge_UpdateStats(self):
volume = db.Volume('db', model.RESOURCES)
cp = NodeRoutes('guid', volume)
- stats = Sniffer(volume)
+ stats = Sniffer(volume, 'stats/node')
records = [
{'resource': 'context'},
@@ -445,28 +445,35 @@ class VolumeTest(tests.Test):
'stability': {'value': 'stable', 'mtime': 1.0},
'notes': {'value': {}, 'mtime': 1.0},
}},
+ {'resource': 'comment'},
+ {'guid': 'comment', 'diff': {
+ 'guid': {'value': 'comment', 'mtime': 1.0},
+ 'ctime': {'value': 1, 'mtime': 1.0},
+ 'mtime': {'value': 1, 'mtime': 1.0},
+ 'context': {'value': 'context', 'mtime': 1.0},
+ 'message': {'value': {}, 'mtime': 1.0},
+ }},
{'commit': [[1, 1]]},
]
merge(volume, records, stats=stats)
ts = int(current_time())
stats.commit(ts)
+ stats.commit_objects()
self.assertEqual([
+ [('comment', ts, {
+ 'total': 1.0,
+ })],
[('feedback', ts, {
- 'solutions': 2.0,
'total': 2.0,
- 'commented': 0.0,
})],
[('review', ts, {
'total': 2.0,
- 'commented': 0.0,
})],
[('solution', ts, {
'total': 2.0,
- 'commented': 0.0,
})],
[('artifact', ts, {
- 'reviewed': 1.0,
'downloaded': 0.0,
'total': 1.0,
})],
@@ -475,7 +482,6 @@ class VolumeTest(tests.Test):
})],
[('context', ts, {
'failed': 0.0,
- 'reviewed': 1.0,
'downloaded': 0.0,
'total': 1.0,
'released': 1.0,
@@ -495,23 +501,22 @@ class VolumeTest(tests.Test):
merge(volume, records, stats=stats)
ts += 1
stats.commit(ts)
+ stats.commit_objects()
self.assertEqual([
+ [('comment', ts, {
+ 'total': 1.0,
+ })],
[('feedback', ts, {
- 'solutions': 1.0,
'total': 2.0,
- 'commented': 0.0,
})],
[('review', ts, {
'total': 2.0,
- 'commented': 0.0,
})],
[('solution', ts, {
'total': 2.0,
- 'commented': 0.0,
})],
[('artifact', ts, {
- 'reviewed': 0.0,
'downloaded': 0.0,
'total': 1.0,
})],
@@ -520,7 +525,6 @@ class VolumeTest(tests.Test):
})],
[('context', ts, {
'failed': 0.0,
- 'reviewed': 0.0,
'downloaded': 0.0,
'total': 1.0,
'released': 0.0,
@@ -549,23 +553,22 @@ class VolumeTest(tests.Test):
merge(volume, records, stats=stats)
ts += 1
stats.commit(ts)
+ stats.commit_objects()
self.assertEqual([
+ [('comment', ts, {
+ 'total': 1.0,
+ })],
[('feedback', ts, {
- 'solutions': 0.0,
'total': 0.0,
- 'commented': 0.0,
})],
[('review', ts, {
'total': 0.0,
- 'commented': 0.0,
})],
[('solution', ts, {
'total': 0.0,
- 'commented': 0.0,
})],
[('artifact', ts, {
- 'reviewed': 0.0,
'downloaded': 0.0,
'total': 0.0,
})],
@@ -574,7 +577,6 @@ class VolumeTest(tests.Test):
})],
[('context', ts, {
'failed': 0.0,
- 'reviewed': 0.0,
'downloaded': 0.0,
'total': 0.0,
'released': 0.0,
diff --git a/tests/units/toolkit/rrd.py b/tests/units/toolkit/rrd.py
index b45b8b4..3434634 100755
--- a/tests/units/toolkit/rrd.py
+++ b/tests/units/toolkit/rrd.py
@@ -277,8 +277,8 @@ class RrdTest(tests.Test):
for i in xrange((end_ts - start_ts) / 300):
dbset.put({'f': i}, start_ts + i * 300)
- prev_ts = 0
- prev_value = 0
+ prev_ts = -1
+ prev_value = -1
for ts, value in dbset.get(start_ts, end_ts, 86400):
value = value['f']
assert ts > prev_ts
generated by cgit v0.9.1 at 2024-04-27 15:08:31 (GMT)