diff options
author | ceibal <ceibal@ceibal.ceibal.uy> | 2013-02-13 14:34:05 (GMT) |
---|---|---|
committer | ceibal <ceibal@ceibal.ceibal.uy> | 2013-02-13 14:34:05 (GMT) |
commit | 28c4581305d5545aadf23132d9d957143a1a71f2 (patch) | |
tree | 2f43a0358b5c674e3eddf0719d2a2a3ebfba5140 | |
parent | b15c5ad3bef164dba29820601215b31de771c2c8 (diff) |
Fixes:
+ Allow update uptime valuo into db, although an activity has an entry already with the same date start.
+ Change last_record update value from last record of las rrd to current time of server
Improves:
+ Enhacement of sql report, add all option.
-rwxr-xr-x | sql/make_report | 55 | ||||
-rwxr-xr-x | sql/report.sh | 3 | ||||
-rw-r--r-- | sugar_stats_consolidation/consolidation.py | 4 | ||||
-rwxr-xr-x | sugar_stats_consolidation/consolidation_run | 5 | ||||
-rw-r--r-- | sugar_stats_consolidation/db.py | 46 | ||||
-rwxr-xr-x[-rw-r--r--] | test/test_cons.py | 0 | ||||
-rwxr-xr-x[-rw-r--r--] | test/test_db.py | 0 | ||||
-rwxr-xr-x[-rw-r--r--] | test/test_rrd.py | 0 | ||||
-rwxr-xr-x[-rw-r--r--] | test/test_rrds_content.py | 4 |
9 files changed, 80 insertions, 37 deletions
diff --git a/sql/make_report b/sql/make_report index 1dae37f..f09b40b 100755 --- a/sql/make_report +++ b/sql/make_report @@ -45,31 +45,46 @@ log.addHandler(ch) logging.getLogger('sugar_stats_consolidation.db').addHandler(ch) +def activity_most_used(re, start, end): + activity = re.most_activity_used(start, end); + log.debug("**************************************************************************************************************") + log.debug(" Most Activity used ") + log.debug('NAME: %s', activity[0]) + log.debug('INTERVAL: %s-> %s', start, end) + log.debug('TIME: %.2f hrs / %.2f mins / %.2f secs', (activity[1]/60/60), (activity[1]/60), activity[1]) + log.debug("***************************************************************************************************************") + +def frequency_usage(re, start, end): + ts_end = long (datetime.strptime(end, "%Y-%m-%d").strftime("%s")) + ts_start = long (datetime.strptime(start, "%Y-%m-%d").strftime("%s")) + + fres = re.frequency_usage(start, end); + f_x_day = (fres / ((ts_end - ts_start)/(60*60*24)))/60 + f_total = (fres / 60) + log.debug("**************************************************************************************************************") + log.debug(" Fequency Usage ") + log.debug('INTERVAL: %s-> %s', start, end) + log.debug('PER DAY: %.2f min / day', (f_x_day)) + log.debug('TOTAL: %.2f min', (f_total)) + log.debug("**************************************************************************************************************") + + def main(): re = DB_Stats (args.db_name, args.db_user, args.db_pass) re.connect() + if args.query == 'activity_most_used': - activity = re.most_activity_used(args.start_date, args.end_date); - log.debug("**************************************************************************************************************") - log.debug(" Most Activity used ") - log.debug('NAME: %s', activity[0]) - log.debug('PERIOD: %s-> %s', args.start_date, args.end_date) - log.debug('TIME: %.2f hrs / %.2f mins / %.2f secs', (activity[1]/60/60), (activity[1]/60), activity[1]) - log.debug("***************************************************************************************************************") - if args.query == 'frequency_usage': - ts_end = long (datetime.strptime(args.end_date, "%Y-%m-%d").strftime("%s")) - ts_start = long (datetime.strptime(args.start_date, "%Y-%m-%d").strftime("%s")) - - fres = re.frequency_usage(args.start_date, args.end_date); - f_x_day = (fres / ((ts_end - ts_start)/(60*60*24)))/60 - f_total = (fres / 60) - log.debug("**************************************************************************************************************") - log.debug(" Fequency Usage ") - log.debug('PERIOD: %s-> %s', args.start_date, args.end_date) - log.debug('USAGE PER DAY: %.2f min / day', (f_x_day)) - log.debug('USAGE TOTAL: %.2f min', (f_total)) - log.debug("**************************************************************************************************************") + activity_most_used(re, args.start_date, args.end_date); + elif args.query == 'frequency_usage': + frequency_usage(re, args.start_date, args.end_date) + elif args.query == 'all': + activity_most_used(re, args.start_date, args.end_date); + frequency_usage(re, args.start_date, args.end_date) + else: + log.error("Query option unkwon: %s", args.query) + + re.close() if __name__ == '__main__': main() diff --git a/sql/report.sh b/sql/report.sh index 5f18985..f14aea3 100755 --- a/sql/report.sh +++ b/sql/report.sh @@ -1,6 +1,7 @@ #!/bin/bash #QUERY=activity_most_used -QUERY=frequency_usage +#QUERY=frequency_usage +QUERY=all START_DATE=2013-02-10 END_DATE=2013-02-14 DB_NAME=statistics diff --git a/sugar_stats_consolidation/consolidation.py b/sugar_stats_consolidation/consolidation.py index b5a980b..8214d9b 100644 --- a/sugar_stats_consolidation/consolidation.py +++ b/sugar_stats_consolidation/consolidation.py @@ -42,7 +42,9 @@ class Consolidation: else: log.warning('RRD file not found: %s', os.path.join(self.base_path, id_hash, user_hash)) else: - log.warning('Hash user direcotory not found: %s', os.path.join(self.base_path, id_hash)) + log.warning('Hash user direcotory not found: %s', os.path.join(self.base_path, id_hash)) + self.db.update_last_record(); + log.info("End RRDs processing") else: log.error('Hash ids not found on: %s', self.base_path) except Exception as e: diff --git a/sugar_stats_consolidation/consolidation_run b/sugar_stats_consolidation/consolidation_run index 46b676d..e955184 100755 --- a/sugar_stats_consolidation/consolidation_run +++ b/sugar_stats_consolidation/consolidation_run @@ -54,6 +54,11 @@ def main(): try: con = Consolidation(args.rrd_path, db) con.process_rrds() + db.close() + + log.info('**************************************') + log.info(' Finish consolidation ') + log.info('**************************************') except Exception as e: log.warning('Processing rrd file: %s', str(e)) diff --git a/sugar_stats_consolidation/db.py b/sugar_stats_consolidation/db.py index 177e157..d387ed7 100644 --- a/sugar_stats_consolidation/db.py +++ b/sugar_stats_consolidation/db.py @@ -106,6 +106,16 @@ class DB_Stats: self.store_user(rrd) cursor = self.cnx.cursor() + select = ("SELECT * FROM Usages WHERE " + "user_hash = %s AND " + "resource_name = %s AND " + "start_date = %s AND " + "data_type = %s") + update = ("UPDATE Usages SET data = %s WHERE " + "user_hash = %s AND " + "resource_name = %s AND " + "start_date = %s AND " + "data_type = %s") insert = ("INSERT INTO Usages " "(user_hash, " "resource_name, " @@ -115,11 +125,22 @@ class DB_Stats: "VALUES (%s, %s, %s, %s ,%s) ") for d in rrd.get_uptime_by_interval(): - info = (rrd.get_user_hash(), rrd.get_name() , datetime.fromtimestamp(float(d[0])), 'uptime', d[1]) + info_sel = (rrd.get_user_hash(), rrd.get_name() , datetime.fromtimestamp(float(d[0])), 'uptime') try: - cursor.execute(insert, info) - if self.update_last_record(rrd.get_date_last_record()) == 0: - self.cnx.commit() + """Verify if this activity has an entry already at the same start_date""" + cursor.execute (select, info_sel) + result = cursor.fetchone() + + if result != None: + log.info('Update uptime \'%s\' entry for resource \'%s\' ', d[1], rrd.get_name()) + info_up = (d[1], rrd.get_user_hash(), rrd.get_name() , datetime.fromtimestamp(float(d[0])), 'uptime') + cursor.execute(update, info_up) + else: + log.info('New uptime \'%s\' entry for resource \'%s\'', d[1], rrd.get_name()) + info_ins = (rrd.get_user_hash(), rrd.get_name() , datetime.fromtimestamp(float(d[0])), 'uptime', d[1]) + cursor.execute(insert, info_ins) + + self.cnx.commit() except mysql.connector.Error as err: log.error('MySQL on store_activiy_uptime: %s %s', cursor.statement, err) @@ -134,7 +155,7 @@ class DB_Stats: cursor.execute(op, params) result = cursor.fetchone() if result != None: - log.info('Resource %s already present in DB', resource_name) + log.debug('Resource %s already present in DB', resource_name) else: insert = ("INSERT INTO Resources (name) VALUES (%s)") info = (resource_name, ) @@ -153,7 +174,7 @@ class DB_Stats: cursor.execute(op, params) result = cursor.fetchone() if result != None: - log.info('User %s already in DB', rrd.user_hash) + log.debug('User %s already in DB', rrd.user_hash) else: insert = ("INSERT INTO Users (hash, uuid, machine_sn, age, school, sw_version) VALUES (%s, %s, %s, %s, %s, %s)") params = (rrd.get_user_hash(), rrd.get_uuid(), rrd.get_sn(), rrd.get_age(), rrd.get_school(), "1.0.0") @@ -166,24 +187,23 @@ class DB_Stats: - def update_last_record (self, ts): + def update_last_record (self): cursor = self.cnx.cursor() res = 0 op = ("SELECT * FROM Runs") - params = (datetime.fromtimestamp(float(ts)),) try: cursor.execute(op) result = cursor.fetchone() if result != None: - op = ("UPDATE Runs SET last_ts = %s") - cursor.execute(op, params) + op = ("UPDATE Runs SET last_ts = CURRENT_TIMESTAMP") + cursor.execute(op) self.cnx.commit() else: - op = ("INSERT INTO Runs VALUES(%s)") - cursor.execute(op, params) + op = ("INSERT INTO Runs VALUES(CURRENT_TIMESTAMP)") + cursor.execute(op) self.cnx.commit() - + log.info("Save last record"); except mysql.connector.Error as err: log.error('MySQL on update_last_record: %s %s', cursor.statement, err) res = -1 diff --git a/test/test_cons.py b/test/test_cons.py index f58f9fc..f58f9fc 100644..100755 --- a/test/test_cons.py +++ b/test/test_cons.py diff --git a/test/test_db.py b/test/test_db.py index 197510a..197510a 100644..100755 --- a/test/test_db.py +++ b/test/test_db.py diff --git a/test/test_rrd.py b/test/test_rrd.py index 81dd3c8..81dd3c8 100644..100755 --- a/test/test_rrd.py +++ b/test/test_rrd.py diff --git a/test/test_rrds_content.py b/test/test_rrds_content.py index df6c99e..134134f 100644..100755 --- a/test/test_rrds_content.py +++ b/test/test_rrds_content.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import argparse -from rrd_files import * +from sugar_stats_consolidation.rrd_files import * parser = argparse.ArgumentParser() parser.add_argument('--rrd_path',required=False) @@ -12,7 +12,7 @@ args = parser.parse_args() print "============================== TEST RRD analyze content ========================================" if args.rrd_path == None: - def_path = "/home/olpc/.sugar/default/stats" + def_path = "/var/lib/sugar-stats/rrd/ed/ed4f8bd4c24d4f10b7bd6c59add7032b0fbf5dbd" else: def_path = args.rrd_path |