fixe, detection of None periods for a while activity still running. Modify rrd_files.py

9 files changed, 90 insertions, 40 deletions

diff --git a/build/lib.linux-x86_64-2.7/stats_consolidation/rrd_files.py b/build/lib.linux-x86_64-2.7/stats_consolidation/rrd_files.py
index c5561d7..5ef1d55 100644
--- a/build/lib.linux-x86_64-2.7/stats_consolidation/rrd_files.py
+++ b/build/lib.linux-x86_64-2.7/stats_consolidation/rrd_files.py
@@ -1,6 +1,7 @@
 import rrdtool
 import os
 import sys
+from datetime import datetime
 
 class RRD:
 
@@ -9,7 +10,7 @@ class RRD:
 	data_item = 2
 	DS = {'active':0, 'buddies':0, 'instances':0, 'new':0, 'resumed':0, 'uptime':0}
 
-	def __init__(self, path, name, date_start, date_end):
+	def __init__(self, path, name, date_start=None, date_end=None):
 
 		self.rrd_name = name
 
@@ -33,9 +34,9 @@ class RRD:
 					self.user_hash[:2],
 					self.user_hash
 					)
-			
+	        """		
 		self.uuid = self.get_uuid_from_file(self.user_path) 
-
+		"""	
 
 		print "*******************************************"
 		print "                  RRD                      "
@@ -67,32 +68,42 @@ class RRD:
 			i=+1
 		return -1
 
-	def get_uptime_by_interval (self):
-		ds_name = "uptime"
+	"""
+	Find several valid record consecutives, the last one is time of the interval.
+	Return: a list (start_time, total_time)
+	"""
+	def get_last_value_by_interval (self, ds_name):
 		res=list()
-	
-		print "-------Calcule "+ ds_name +"-------"
+		prev_value = 0.0
 		i=0
 		found = False
+
+		print "-------Calcule "+ ds_name +"-------"
 		while i < len(self.rrd[self.data_item]):
-			value     = str(self.rrd[self.data_item][i][self.DS[ds_name]])
-			if value != "None":
-				uptime = value
-				end    = str (long(self.date_start) + ((i+1) * 60))
+			value  = str(self.rrd[self.data_item][i][self.DS[ds_name]])
+
+			if (value != "None") and (float(value) > 0)  and (float(value) > float(prev_value)):
+				prev_value = value
+				end    = long(self.date_start) + ((i+1) * 60)
 				if found == False:
 					found = True
-					start = str (long (self.date_start) + ((i+1) * 60))
+					start = long (self.date_start) + ((i+1) * 60)
 			else:
 				if found:
-					print start + "->" + end + ": " + uptime
-					if float(uptime) > 0:
-						res.append((start, uptime))
+					print str(datetime.fromtimestamp(float(start))) + " -> " + str(datetime.fromtimestamp(float(end))) + ": " + prev_value
+					res.append((start, prev_value))
 					found = False
+					prev_value = 0.0
 			i=i+1
 		return res
 		print "---------------------------------------------------"
+	
+	def get_active_by_interval (self):
+		return self.get_last_value_by_interval ("active")
 
-
+	def get_uptime_by_interval (self):
+		return self.get_last_value_by_interval ("uptime")
+	
 	def get_name(self):
 		return self.rrd_name.partition(".rrd")[0]
 
@@ -104,7 +115,7 @@ class RRD:
 			value     = str (self.rrd[self.data_item][i][self.DS[ds_name]])
 	
 			if value != "None":
-				print timestamp+ ": " + value
+				print str(datetime.fromtimestamp(float(timestamp))) + " (" + timestamp + ")" + ": " + value
 			i=i+1
 		print "---------------------------------------------------"
 
diff --git a/build/lib.linux-x86_64-2.7/stats_consolidation/test_rrd.py b/build/lib.linux-x86_64-2.7/stats_consolidation/test_rrd.py
index 657ce18..81dd3c8 100644
--- a/build/lib.linux-x86_64-2.7/stats_consolidation/test_rrd.py
+++ b/build/lib.linux-x86_64-2.7/stats_consolidation/test_rrd.py
@@ -1,8 +1,15 @@
-from db import *
+#!/usr/bin/env python
 
+import argparse
+from db import *
 from rrd_files import *
 from db import *
 
+parser = argparse.ArgumentParser()
+parser.add_argument('--rrd_path',required=False)
+parser.add_argument('--rrd_name',required=True)
+
+args = parser.parse_args()
 
 print "==============================       TEST RRD -> Relational DB  ========================================"
 db = DB_Stats('statistics', 'root', 'gustavo')
@@ -26,13 +33,20 @@ DATE_END = datetime(year=2012,
 DATE_START = db.get_date_last_record()
 DATE_END = datetime.now().strftime("%s")
 
-act_rrd = RRD (path = "/home/gustavo/AC/consolidation/rrds", name="pippy.rrd", date_start=DATE_START, date_end=DATE_END)
+act_rrd = RRD (path = "/home/gustavo/AC/consolidation/rrds", name=args.rrd_name)
+"""
+act_rrd = RRD (path = "/home/gustavo/AC/consolidation/rrds", name="abacus.rrd", date_start=DATE_START, date_end=DATE_END)
+"""
 """
-act_rrd.show_valid_ds("uptime")
 act_rrd.show_valid_ds("resumed")
 act_rrd.show_valid_ds("new")
 act_rrd.show_valid_ds("instances")
 act_rrd.show_valid_ds("buddies")
-"""
 data = {}
 db.store_activity_uptime(act_rrd)
+"""
+act_rrd.show_valid_ds("uptime")
+act_rrd.show_valid_ds("active")
+
+act_rrd.get_uptime_by_interval()
+act_rrd.get_active_by_interval()
diff --git a/rrds/abacus.rrd b/rrds/abacus.rrd
index 32d5cf0..89aaaf0 100644
--- a/rrds/abacus.rrd
+++ b/rrds/abacus.rrd
diff --git a/rrds/journal.rrd b/rrds/journal.rrd
deleted file mode 100644
index 889b25f..0000000
--- a/rrds/journal.rrd
+++ /dev/null
diff --git a/rrds/pippy.rrd b/rrds/pippy.rrd
deleted file mode 100644
index 3f1e373..0000000
--- a/rrds/pippy.rrd
+++ /dev/null
diff --git a/setup.py b/setup.py
index 3a2e344..ed8be24 100644
--- a/setup.py
+++ b/setup.py
@@ -8,8 +8,8 @@ setup(
     author_email = "gduarte@activitycentral.com",
     url = "http://www.acrtivitycentral.com/",
     packages=[
-        'stats_consolidation',
+        'sugar-stats-consolidation',
     ],
-    package_dir={'': 'src'}
+    package_dir={'': 'stats_consolidation'}
 )
 
diff --git a/sql/report.py b/sql/report.py
index e3375a5..f9a600d 100644
--- a/sql/report.py
+++ b/sql/report.py
@@ -24,9 +24,9 @@ class Report:
 	STAT={}
 	STAT ['Get_resource_name'] = ("SELECT name FROM `Resources`")
 
-	STAT ['Get_suma_uptime'] = ( "SELECT `data` FROM Usages (WHERE `resource_name` = %s")
+	STAT ['Get_suma_uptime'] = ( "SELECT SUM (`data`) FROM Usages (WHERE `resource_name` = %s" AND data_type = 'active')
 
-	STAT ['Get_frequency_usage'] = ("SELECT SUM(`data`) FROM Usages (Where `resource_name` = `system`)")
+	STAT ['Get_frequency_usage'] = ("SELECT SUM(`data`) FROM Usages ((WHERE `resource_name` = `system`) AND (start_date > start) AND (start_date < end))")
 	
 
 	def __init__ (self,  db_name, user, password):
diff --git a/sugar_stats_consolidation/rrd_files.py b/sugar_stats_consolidation/rrd_files.py
index 5ef1d55..171dbe9 100644
--- a/sugar_stats_consolidation/rrd_files.py
+++ b/sugar_stats_consolidation/rrd_files.py
@@ -82,22 +82,24 @@ class RRD:
 		while i < len(self.rrd[self.data_item]):
 			value  = str(self.rrd[self.data_item][i][self.DS[ds_name]])
 
-			if (value != "None") and (float(value) > 0)  and (float(value) > float(prev_value)):
+			if (value != "None") and (float(value) > 0)  and (float(value) >= float(prev_value)):
 				prev_value = value
-				end    = long(self.date_start) + ((i+1) * 60)
+				end = long(self.date_start) + ((i+1) * 60)
 				if found == False:
 					found = True
 					start = long (self.date_start) + ((i+1) * 60)
 			else:
 				if found:
-					print str(datetime.fromtimestamp(float(start))) + " -> " + str(datetime.fromtimestamp(float(end))) + ": " + prev_value
-					res.append((start, prev_value))
-					found = False
-					prev_value = 0.0
+					if self.verify_interrupt(i, ds_name, prev_value):
+						print str(datetime.fromtimestamp(float(start))) + " -> " + str(datetime.fromtimestamp(float(end))) + ": " + prev_value
+						res.append((start, prev_value))
+						found = False
+						prev_value = 0.0
 			i=i+1
 		return res
 		print "---------------------------------------------------"
-	
+
+
 	def get_active_by_interval (self):
 		return self.get_last_value_by_interval ("active")
 
@@ -142,3 +144,28 @@ class RRD:
 
 	def get_uuid (self):
 		return self.uuid
+
+	"""
+	For some reason, sometimes for a while activity is running, statistics library register several values as None.
+	To detect this behavoir, this function look-up over next records time, and verify if the value is grater than
+	last valid value + (interval_numb * 60). If the value es greater, means the activity still running else 
+        the activity was stopped and starting again. 
+	"""
+	def verify_interrupt(self, idx, ds_name, prev_value):
+		i = idx
+		j = 0
+		while i < len(self.rrd[self.data_item]):
+			value  = str(self.rrd[self.data_item][i][self.DS[ds_name]])
+			if value != "None":
+				"""
+				print "["+str(j)+ "] current value: " + value + " prev value: " +  str (float (prev_value) + (60 * j)) + " ("+ prev_value+")"
+				"""
+				if float(value) > (float (prev_value) + (60 * j)):
+					return False
+				else:
+					return True
+			i=i+1
+			j=j+1
+
+		return True
+	
diff --git a/sugar_stats_consolidation/test_rrds_content.py b/sugar_stats_consolidation/test_rrds_content.py
index 959b618..df6c99e 100755
--- a/sugar_stats_consolidation/test_rrds_content.py
+++ b/sugar_stats_consolidation/test_rrds_content.py
@@ -1,9 +1,7 @@
 #!/usr/bin/env python
 
 import argparse
-from db import *
 from rrd_files import *
-from db import *
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--rrd_path',required=False)
@@ -14,11 +12,11 @@ args = parser.parse_args()
 print "==============================       TEST RRD analyze content  ========================================"
 
 if args.rrd_path == None:
-	def_path = "/home/gustavo/AC/consolidation/rrds"
+	def_path = "/home/olpc/.sugar/default/stats"
 else: 
 	def_path = args.rrd_path
 
-act_rrd = RRD (path=def_path, name=args.rrd_name)
+rrd = RRD (path=def_path, name=args.rrd_name)
 
 """
 act_rrd.show_valid_ds("resumed")
@@ -26,8 +24,8 @@ act_rrd.show_valid_ds("new")
 act_rrd.show_valid_ds("instances")
 act_rrd.show_valid_ds("buddies")
 """
-act_rrd.show_valid_ds("uptime")
-act_rrd.show_valid_ds("active")
+rrd.show_valid_ds("uptime")
+rrd.show_valid_ds("active")
 
-act_rrd.get_uptime_by_interval()
-act_rrd.get_active_by_interval()
+rrd.get_uptime_by_interval()
+rrd.get_active_by_interval()