Avoid using frontends in server code to make them as pluggable as possibleserver-votes.merged

16 files changed, 415 insertions, 74 deletions

diff --git a/TODO b/TODO
index e4ea54a..3a0b406 100644
--- a/TODO
+++ b/TODO
@@ -2,6 +2,7 @@
 ===
 - find() for multiple resources
 - use Artifact.mime_type instead of Artifact.type
+- "collectioins" of contexts, e.g., to provide featured activities
 
 1.0
 ===
diff --git a/sugar_network_server/__init__.py b/sugar_network_server/__init__.py
index 42143eb..32ff9cf 100644
--- a/sugar_network_server/__init__.py
+++ b/sugar_network_server/__init__.py
@@ -17,9 +17,9 @@ from active_document import data_root, index_flush_timeout, \
         index_flush_threshold, index_write_queue, find_limit
 
 from restful_document import host, port, debug, foreground, logdir, rundir, \
-        trust_users, auth, keyfile, certfile, master
+        trust_users, keyfile, certfile, master
 
-from sugar_stats_server import stats, stats_root, stats_step, \
+from sugar_network_server.env import stats, stats_root, stats_step, \
         stats_server_rras, stats_client_rras
 
 from sugar_network_server.resources import resources
@@ -28,6 +28,10 @@ from sugar_network_server.resources import resources
 def config(**kwargs):
     from gettext import gettext as _
     import restful_document
+    import active_document
+    from sugar_network_server.resources import env
+
+    active_document.util.Option.seek('stats', env)
 
     return restful_document.config(
             name='sugar-network-server',
diff --git a/sugar_network_server/env.py b/sugar_network_server/env.py
index cb70623..20c5071 100644
--- a/sugar_network_server/env.py
+++ b/sugar_network_server/env.py
@@ -13,6 +13,11 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from gettext import gettext as _
+
+import active_document as ad
+util = ad.util
+
 
 VERSION = '0.1'
 
@@ -29,3 +34,28 @@ STABILITIES = [
 ARTIFACT_TYPES = [
         'screenshot', 'jobject',
         ]
+
+
+stats = util.Option(
+        _('enable stats collecting'),
+        default=True, type_cast=util.Option.bool_cast, action='store_true')
+
+stats_root = util.Option(
+        _('path to the root directory for placing stats'),
+        default='/var/lib/sugar-network/stats')
+
+stats_step = util.Option(
+        _('step interval in seconds for RRD databases'),
+        default=60, type_cast=int)
+
+stats_server_rras = util.Option(
+        _('space separated list of RRAs for RRD databases on a server side'),
+        default='RRA:AVERAGE:0.5:1:4320 RRA:AVERAGE:0.5:5:2016',
+        type_cast=lambda x: [i for i in x.split() if i],
+        type_repr=lambda x: ' '.join(x))
+
+stats_client_rras = util.Option(
+        _('space separated list of RRAs for RRD databases on client side'),
+        default='RRA:AVERAGE:0.5:1:4320 RRA:AVERAGE:0.5:5:2016',
+        type_cast=lambda x: [i for i in x.split() if i],
+        type_repr=lambda x: ' '.join(x))
diff --git a/sugar_network_server/resources/artifact.py b/sugar_network_server/resources/artifact.py
index 7a2c7ab..3dfb5bc 100644
--- a/sugar_network_server/resources/artifact.py
+++ b/sugar_network_server/resources/artifact.py
@@ -52,6 +52,6 @@ class Artifact(Resource):
     def rating(self, value):
         return value
 
-    @ad.active_property(Vote, counter='rating')
+    @ad.active_property(Vote)
     def vote(self, value):
         return value
diff --git a/sugar_network_server/resources/context.py b/sugar_network_server/resources/context.py
index c385fd8..d8fff65 100644
--- a/sugar_network_server/resources/context.py
+++ b/sugar_network_server/resources/context.py
@@ -76,6 +76,6 @@ class Context(Resource):
     def rating(self, value):
         return value
 
-    @ad.active_property(Vote, counter='rating')
+    @ad.active_property(Vote)
     def vote(self, value):
         return value
diff --git a/sugar_network_server/resources/idea.py b/sugar_network_server/resources/idea.py
index e4ef70a..f23c092 100644
--- a/sugar_network_server/resources/idea.py
+++ b/sugar_network_server/resources/idea.py
@@ -37,6 +37,6 @@ class Idea(Resource):
     def rating(self, value):
         return value
 
-    @ad.active_property(Vote, counter='rating')
+    @ad.active_property(Vote)
     def vote(self, value):
         return value
diff --git a/sugar_network_server/resources/implementation.py b/sugar_network_server/resources/implementation.py
index a766375..b752890 100644
--- a/sugar_network_server/resources/implementation.py
+++ b/sugar_network_server/resources/implementation.py
@@ -62,17 +62,18 @@ class Implementation(Resource):
     def bundle(self, value):
         return value
 
-    def send_blob(self, prop):
-        from sugar_network_server import resources
+    def get_blob(self, prop_name, raw=False):
+        if prop_name == 'bundle':
+            from sugar_network_server import resources
 
-        context = resources.Context(self['context'])
-        feed_data = context.get_blob('feed')
+            context = resources.Context(self['context'])
+            feed_data = context.get_blob('feed')
 
-        if feed_data is not None:
-            feed = json.load(feed_data)
-            if self['version'] in feed:
-                url = feed[self['version']]['*-*'].get('url')
-                if url:
-                    raise rd.SeeOther(url)
+            if feed_data is not None:
+                feed = json.load(feed_data)
+                if self['version'] in feed:
+                    url = feed[self['version']]['*-*'].get('url')
+                    if url:
+                        raise rd.SeeOther(url)
 
-        return Resource.send_blob(self, prop)
+        return Resource.send_blob(self, prop_name, raw)
diff --git a/sugar_network_server/resources/problem.py b/sugar_network_server/resources/problem.py
index 1f63c13..68807d4 100644
--- a/sugar_network_server/resources/problem.py
+++ b/sugar_network_server/resources/problem.py
@@ -41,6 +41,6 @@ class Problem(Resource):
     def rating(self, value):
         return value
 
-    @ad.active_property(Vote, counter='rating')
+    @ad.active_property(Vote)
     def vote(self, value):
         return value
diff --git a/sugar_network_server/resources/question.py b/sugar_network_server/resources/question.py
index aafd6c4..5e4eeb2 100644
--- a/sugar_network_server/resources/question.py
+++ b/sugar_network_server/resources/question.py
@@ -41,6 +41,6 @@ class Question(Resource):
     def rating(self, value):
         return value
 
-    @ad.active_property(Vote, counter='rating')
+    @ad.active_property(Vote)
     def vote(self, value):
         return value
diff --git a/sugar_network_server/resources/resource.py b/sugar_network_server/resources/resource.py
index 4777f00..ba13e67 100644
--- a/sugar_network_server/resources/resource.py
+++ b/sugar_network_server/resources/resource.py
@@ -13,39 +13,19 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from gettext import gettext as _
-
 import active_document as ad
-import restful_document as rd
 
 enforce = ad.util.enforce
 
 
-class Resource(rd.Document):
-
-    @ad.active_property(prefix='OA', typecast=[])
-    def author(self, value):
-        return value
+class Resource(ad.Document):
 
     @ad.active_property(prefix='OT', default=[], typecast=[])
     def tags(self, value):
         return value
 
-    @rd.restful_method(method='PUT')
-    def restful_put(self, prop=None, url=None):
-        enforce(rd.principal.user in self['author'],
-                rd.Forbidden, _('Access is not permitted for not authors'))
-        rd.Document.restful_put(self, prop, url)
-
-    @rd.restful_method(method='DELETE')
-    def restful_delete(self, prop=None):
-        enforce(rd.principal.user in self['author'],
-                rd.Forbidden, _('Access is not permitted for not authors'))
-        rd.Document.restful_delete(self, prop)
-
 
 class Vote(ad.AggregatorProperty):
 
-    @property
-    def value(self):
-        return rd.principal.user
+    def __init__(self, *args):
+        ad.AggregatorProperty.__init__(self, 'vote', counter='rating')
diff --git a/sugar_network_server/resources/review.py b/sugar_network_server/resources/review.py
index 34caf0e..a214ae2 100644
--- a/sugar_network_server/resources/review.py
+++ b/sugar_network_server/resources/review.py
@@ -33,6 +33,6 @@ class Review(Resource):
     def rating(self, value):
         return value
 
-    @ad.active_property(Vote, counter='rating')
+    @ad.active_property(Vote)
     def vote(self, value):
         return value
diff --git a/sugar_network_server/resources/root.py b/sugar_network_server/resources/root.py
deleted file mode 100644
index d85091e..0000000
--- a/sugar_network_server/resources/root.py
+++ /dev/null
@@ -1,28 +0,0 @@
-# Copyright (C) 2012, Aleksey Lim
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-from gettext import gettext as _
-
-
-import restful_document as rd
-
-
-class Root(rd.Document):
-
-    @staticmethod
-    @rd.restful_method(method='GET')
-    def _restful_hello():
-        rd.responce['Content-Type'] = 'text/plain'
-        return _('Hello, world!')
diff --git a/sugar_network_server/resources/solution.py b/sugar_network_server/resources/solution.py
index 2f8da7e..9bb71de 100644
--- a/sugar_network_server/resources/solution.py
+++ b/sugar_network_server/resources/solution.py
@@ -40,6 +40,6 @@ class Solution(Resource):
     def rating(self, value):
         return value
 
-    @ad.active_property(Vote, counter='rating')
+    @ad.active_property(Vote)
     def vote(self, value):
         return value
diff --git a/sugar_network_server/resources/user.py b/sugar_network_server/resources/user.py
index 40aef07..7ab8c57 100644
--- a/sugar_network_server/resources/user.py
+++ b/sugar_network_server/resources/user.py
@@ -13,11 +13,42 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
+import hashlib
+from os.path import join
+from gettext import gettext as _
+
+import restful_document as rd
 import active_document as ad
-from sugar_stats_server import User as StatsUser
+enforce = ad.util.enforce
+
+from sugar_network_server import env
+from sugar_network_server.rrd import Rrd
+
+
+class User(ad.Document):
+
+    _rrd = {}
+
+    @ad.active_property(slot=100, prefix='N', full_text=True)
+    def nickname(self, value):
+        return value
+
+    @ad.active_property(ad.StoredProperty)
+    def color(self, value):
+        return value
+
+    @ad.active_property(slot=101, prefix='S', permissions=ad.ACCESS_CREATE)
+    def machine_sn(self, value):
+        return value
 
+    @ad.active_property(slot=102, prefix='U', permissions=ad.ACCESS_CREATE)
+    def machine_uuid(self, value):
+        return value
 
-class User(StatsUser):
+    @ad.active_property(ad.StoredProperty, permissions=ad.ACCESS_CREATE)
+    def pubkey(self, value):
+        return value
 
     @ad.active_property(prefix='T', full_text=True, default=[], typecast=[])
     def tags(self, value):
@@ -34,3 +65,62 @@ class User(StatsUser):
     @ad.active_property(slot=3, prefix='B', default=0, typecast=int)
     def birthday(self, value):
         return value
+
+    @classmethod
+    def create(cls, properties):
+        enforce('pubkey' in properties,
+                _('Property "pubkey" is required for user registeration'))
+        guid, properties['pubkey'] = _load_pubkey(properties['pubkey'].strip())
+        doc = cls(**(properties or {}))
+        doc.set('guid', guid, raw=True)
+        doc.post()
+        return doc
+
+    @ad.active_method(cmd='stats-info',
+            permissions=ad.ACCESS_AUTHOR)
+    def _stats_info(self):
+        status = {}
+        rrd = User._get_rrd(self.guid)
+        for name, __, last_update in rrd.dbs:
+            status[name] = last_update + env.stats_step.value
+        return {'enable': env.stats.value,
+                'step': env.stats_step.value,
+                'rras': env.stats_client_rras.value,
+                'status': status,
+                }
+
+    @ad.active_method(http_method='POST', cmd='stats-upload',
+            permissions=ad.ACCESS_AUTHOR)
+    def _stats_upload(self, name, values):
+        rrd = User._get_rrd(self.guid)
+        for timestamp, values in values:
+            rrd.put(name, values, timestamp)
+
+    @classmethod
+    def _get_rrd(cls, guid):
+        rrd = cls._rrd.get(guid)
+        if rrd is None:
+            rrd = cls._rrd[guid] = Rrd(
+                    join(env.stats_root.value, guid[:2], guid),
+                    env.stats_step.value, env.stats_server_rras.value)
+        return rrd
+
+
+def _load_pubkey(pubkey):
+    try:
+        src_path = ad.util.TempFilePath(text=pubkey)
+        # SSH key needs to be converted to PKCS8 to ket M2Crypto read it
+        pubkey_pkcs8 = ad.util.assert_call(
+                ['ssh-keygen', '-f', src_path, '-e', '-m', 'PKCS8'])
+    except Exception:
+        message = _('Cannot read DSS public key gotten for registeration')
+        ad.util.exception(message)
+        if rd.trust_users.value:
+            logging.warning(_('Failed to read registration pubkey, ' \
+                    'but we trust users'))
+            # Keep SSH key for further converting to PKCS8
+            pubkey_pkcs8 = pubkey
+        else:
+            raise rd.Forbidden(message)
+
+    return str(hashlib.sha1(pubkey.split()[1]).hexdigest()), pubkey_pkcs8
diff --git a/sugar_network_server/rrd.py b/sugar_network_server/rrd.py
new file mode 100644
index 0000000..23c2e61
--- /dev/null
+++ b/sugar_network_server/rrd.py
@@ -0,0 +1,263 @@
+# Copyright (C) 2012, Aleksey Lim
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Convenient access to RRD databases.
+
+$Repo: git://git.sugarlabs.org/alsroot/codelets.git$
+$File: src/rrd.py$
+$Data: 2012-04-16$
+
+"""
+
+import re
+import os
+import time
+import bisect
+import logging
+from datetime import datetime
+from os.path import exists, join
+from gettext import gettext as _
+
+import rrdtool
+
+
+_DB_FILENAME_RE = re.compile('(.*?)(-[0-9]+){0,1}\.rrd$')
+_INFO_RE = re.compile('([^[]+)\[([^]]+)\]\.(.*)$')
+
+_FETCH_PAGE = 256
+
+_logger = logging.getLogger('sugar_stats')
+
+
+class Rrd(object):
+
+    def __init__(self, root, step, rras):
+        self._root = root
+        self._step = step
+        # rrdtool knows nothing about `unicode`
+        self._rras = [i.encode('utf8') for i in rras]
+        self._dbsets = {}
+
+        if not exists(self._root):
+            os.makedirs(self._root)
+
+        for filename in os.listdir(self._root):
+            match = _DB_FILENAME_RE.match(filename)
+            if match is not None:
+                name, revision = match.groups()
+                self._dbset(name).load(filename, int(revision or 0))
+
+    @property
+    def step(self):
+        return self._step
+
+    @property
+    def dbs(self):
+        for name, dbset in self._dbsets.items():
+            db = dbset.db
+            if db is not None:
+                yield name, db.first, db.last_update
+
+    def put(self, name, values, timestamp=None):
+        self._dbset(name).put(values, timestamp)
+
+    def get(self, name, start=None, end=None):
+        return self._dbset(name).get(start, end)
+
+    def _dbset(self, name):
+        db = self._dbsets.get(name)
+        if db is None:
+            db = self._dbsets[name] = \
+                    _DbSet(self._root, name, self._step, self._rras)
+        return db
+
+
+class _DbSet(object):
+
+    def __init__(self, root, name, step, rras):
+        self._root = root
+        self._name = name
+        self._step = step
+        self._rras = rras
+        self._revisions = []
+        self._field_names = []
+        self.__db = None
+
+    @property
+    def db(self):
+        if self._revisions:
+            return self._revisions[-1]
+
+    def load(self, filename, revision):
+        _logger.debug('Load %s database from %s with revision %s',
+                filename, self._root, revision)
+        db = _Db(join(self._root, filename), revision)
+        bisect.insort(self._revisions, db)
+        return db
+
+    def put(self, values, timestamp=None):
+        if not self._field_names:
+            self._field_names = values.keys()
+            self._field_names.sort()
+
+        if not timestamp:
+            timestamp = int(time.mktime(datetime.utcnow().utctimetuple()))
+        timestamp = timestamp / self._step * self._step
+
+        db = self._get_db(timestamp)
+        if db is None:
+            return
+
+        if timestamp <= db.last_update:
+            _logger.warning(_('Database %s updated at %s, %s in the past'),
+                    db.path, db.last_update, timestamp)
+            return
+
+        value = [str(timestamp)]
+        for name in self._field_names:
+            value.append(str(values[name]))
+
+        _logger.debug('Put %r to %s', value, db.path)
+
+        db.put(':'.join(value))
+
+    def get(self, start=None, end=None):
+        if not self._revisions:
+            return
+
+        if start is None:
+            start = self._revisions[0].first
+        if end is None:
+            end = self._revisions[-1].last_update
+
+        revisions = []
+        for db in reversed(self._revisions):
+            revisions.append(db)
+            if db.last_update <= start:
+                break
+
+        start = start / self._step * self._step - self._step
+        end = end / self._step * self._step - self._step
+
+        for db in reversed(revisions):
+            db_end = min(end, db.last_update - self._step)
+            while start <= db_end:
+                until = max(start,
+                        min(start + _FETCH_PAGE, db_end))
+                (row_start, start, row_step), __, rows = rrdtool.fetch(
+                        str(db.path),
+                        'AVERAGE',
+                        '--start', str(start),
+                        '--end', str(until))
+                for raw_row in rows:
+                    row_start += row_step
+                    row = {}
+                    accept = False
+                    for i, value in enumerate(raw_row):
+                        row[db.field_names[i]] = value
+                        accept = accept or value is not None
+                    if accept:
+                        yield row_start, row
+                start = until + 1
+
+    def _get_db(self, timestamp):
+        if self.__db is None and self._field_names:
+            if self._revisions:
+                db = self._revisions[-1]
+                if db.last_update >= timestamp:
+                    _logger.warning(
+                            _('Database %s updated at %s, %s in the past'),
+                            db.path, db.last_update, timestamp)
+                    return None
+                if db.step != self._step or db.rras != self._rras or \
+                        db.field_names != self._field_names:
+                    db = self._create_db(self._field_names, db.revision + 1,
+                            db.last_update)
+            else:
+                db = self._create_db(self._field_names, 0, timestamp)
+            self.__db = db
+        return self.__db
+
+    def _create_db(self, field_names, revision, timestamp):
+        filename = self._name
+        if revision:
+            filename += '-%s' % revision
+        filename += '.rrd'
+
+        _logger.debug('Create %s database in %s starting from %s',
+                filename, self._root, timestamp)
+
+        fields = []
+        for name in field_names:
+            fields.append(str('DS:%s:GAUGE:%s:U:U' % (name, self._step * 2)))
+
+        rrdtool.create(
+                str(join(self._root, filename)),
+                '--start', str(timestamp - self._step),
+                '--step', str(self._step),
+                *(fields + self._rras))
+
+        return self.load(filename, revision)
+
+
+class _Db(object):
+
+    def __init__(self, path, revision=0):
+        self.path = str(path)
+        self.revision = revision
+        self.fields = []
+        self.field_names = []
+        self.rras = []
+
+        info = rrdtool.info(self.path)
+        self.step = info['step']
+        self.last_update = info['last_update']
+
+        fields = {}
+        rras = {}
+
+        for key, value in info.items():
+            match = _INFO_RE.match(key)
+            if match is None:
+                continue
+            prefix, key, prop = match.groups()
+            if prefix == 'ds':
+                fields.setdefault(key, {})
+                fields[key][prop] = value
+            if prefix == 'rra':
+                rras.setdefault(key, {})
+                rras[key][prop] = value
+
+        for index in sorted([int(i) for i in rras.keys()]):
+            rra = rras[str(index)]
+            self.rras.append(
+                    'RRA:%(cf)s:%(xff)s:%(pdp_per_row)s:%(rows)s' % rra)
+
+        for name in sorted(fields.keys()):
+            props = fields[name]
+            props['name'] = name
+            self.fields.append(props)
+            self.field_names.append(name)
+
+    def put(self, value):
+        rrdtool.update(self.path, str(value))
+        self.last_update = rrdtool.info(self.path)['last_update']
+
+    @property
+    def first(self):
+        return rrdtool.first(self.path)
+
+    def __cmp__(self, other):
+        return cmp(self.revision, other.revision)
diff --git a/sweets.recipe b/sweets.recipe
index ee2ec9b..132802b 100644
--- a/sweets.recipe
+++ b/sweets.recipe
@@ -11,7 +11,7 @@ homepage  = http://wiki.sugarlabs.org/go/Platform_Team/Sugar_Network/Server
 version   = 0.1
 stability = developer
 
-requires  = active-document; restful-document; sugar-stats-server; sweets-recipe
+requires  = active-document; restful-document; sweets-recipe; base/rrdtool-python
 install   = getent group sugar-network || groupadd -r sugar-network;
             getent passwd sugar-network || useradd -r -d / -M -g sugar-network sugar-network;
             mkdir -p /var/lib/sugar-network /var/log/sugar-network /var/run/sugar-network;