fix on consolidation.py, rrdtool.fetch over one rrd file fails and interrupt the consolidaton procedure.

improve datetime display.

5 files changed, 44 insertions, 34 deletions

diff --git a/setup.py b/setup.py
index ed8be24..e3b6440 100644
--- a/setup.py
+++ b/setup.py
@@ -8,8 +8,8 @@ setup(
     author_email = "gduarte@activitycentral.com",
     url = "http://www.acrtivitycentral.com/",
     packages=[
-        'sugar-stats-consolidation',
+        'sugar_stats_consolidation',
     ],
-    package_dir={'': 'stats_consolidation'}
+    package_dir={'': ''}
 )
 
diff --git a/sugar_stats_consolidation/consolidation.py b/sugar_stats_consolidation/consolidation.py
index a1fd79a..ce72ce7 100644
--- a/sugar_stats_consolidation/consolidation.py
+++ b/sugar_stats_consolidation/consolidation.py
@@ -25,14 +25,17 @@ class Consolidation:
 							if rrd_list:
 								for rrd in rrd_list: 
 									rrd_path = unicode (os.path.join(self.base_path, id_hash, user_hash) )
-									rrd_obj = RRD (path=rrd_path, name=rrd, date_start=self.date_start, date_end=None)
-									self.db.store_activity_uptime(rrd_obj)
+									try:
+										rrd_obj = RRD (path=rrd_path, name=rrd, date_start=self.date_start, date_end=None)
+										self.db.store_activity_uptime(rrd_obj)
+									except Exception as e:
+										print ("warning, coninute ..")
 							else:
-								print ("RRD file not found: {}".format(os.path.join(self.base_path, id_hash, user_hash)))
+								print ("RRD file not found: {0}".format(os.path.join(self.base_path, id_hash, user_hash)))
 					else:
-						print ("None hash user found on: {}".format(os.path.join(self.base_path, id_hash)))	
+						print ("None hash user found on: {0}".format(os.path.join(self.base_path, id_hash)))	
 			else:
-				print ("None hash ids  found on: {}" + format(self.base_path))
+				print ("None hash ids  found on: {0}" + format(self.base_path))
 		except Exception as e:
-			print ("Error: {}".format(str(e)))
+			print ("Error processing rrds: {0}".format(str(e)))
 						
diff --git a/sugar_stats_consolidation/db.py b/sugar_stats_consolidation/db.py
index 33b42c3..de66e2c 100644
--- a/sugar_stats_consolidation/db.py
+++ b/sugar_stats_consolidation/db.py
@@ -3,7 +3,7 @@ import mysql.connector
 from mysql.connector import errorcode
 from datetime import datetime
 
-import stats_consolidation.rrd_files
+import sugar_stats_consolidation.rrd_files
 
 from rrd_files import *
 
@@ -55,20 +55,20 @@ class DB_Stats:
 	def create_database(self, cursor):
 		try:
 			cursor.execute(
-			"CREATE DATABASE {} DEFAULT CHARACTER SET 'utf8'".format(self.db_name))
+			"CREATE DATABASE {0} DEFAULT CHARACTER SET 'utf8'".format(self.db_name))
 		except mysql.connector.Error as err:
-			raise Exception ("Failed creating database: {}".format(err))
+			raise Exception ("Failed creating database: {0}".format(err))
 	
 	def create_tables(self, cursor):
 		for name, ddl in self.TABLES.iteritems():
 			try:
-				print("Creating table {}: ".format(name), end='')
+				print("Creating table {0}: ".format(name), end='')
 				cursor.execute(ddl)
 		    	except mysql.connector.Error as err:
 				if err.errno == errorcode.ER_TABLE_EXISTS_ERROR:
 			    		print("already exists.")
 				else:
-			    		raise Exception ("Error: {}".format(err))
+			    		raise Exception ("Error: {0}".format(err))
 		    	else:
 				print("OK")
 
@@ -87,7 +87,7 @@ class DB_Stats:
 		    		self.cnx.database = self.db_name
 				self.create_tables(cursor)
 		    	else:
-				raise Exception ("Error: {}".format(err))
+				raise Exception ("Error: {0}".format(err))
 		cursor.close()
 	
 	
@@ -120,7 +120,7 @@ class DB_Stats:
 					self.cnx.commit()
 
 			except mysql.connector.Error as err:
-                        	print("Fail {}: {}".format(cursor.statement, err))
+                        	print("Fail {0}: {1}".format(cursor.statement, err))
 		cursor.close()
 	
 
@@ -132,14 +132,14 @@ class DB_Stats:
 			cursor.execute(op, params)
 			result = cursor.fetchone()
 			if result != None:
-				print("Resource {} already in db".format(resource_name))
+				print("Resource {0} already in db".format(resource_name))
 			else:
 				insert = ("INSERT INTO Resources (name) VALUES (%s)")
 				info = (resource_name, )
 				cursor.execute(insert, info)
 				self.cnx.commit()
 		except mysql.connector.Error as err:
-                        print("Fail {}: {}".format(cursor.statement, err))
+                        print("Fail {0}: {1}".format(cursor.statement, err))
 
 		cursor.close()
 
@@ -151,7 +151,7 @@ class DB_Stats:
                         cursor.execute(op, params)
                         result = cursor.fetchone()
                         if result != None:
-                                print("User {} already in db".format(rrd.user_hash))
+                                print("User {0} already in db".format(rrd.user_hash))
                         else:
 				"""FIXME change hardcoded values """
                                 insert = ("INSERT INTO Users (hash, uuid, machine_sn, age, school, sw_version) VALUES (%s, %s, %s, %s, %s, %s)")
@@ -159,7 +159,7 @@ class DB_Stats:
                                 cursor.execute(insert, params)
                                 self.cnx.commit()
                 except mysql.connector.Error as err:
-                        print("Fail {}: {}".format(cursor.statement, err))
+                        print("Fail {0}: {1}".format(cursor.statement, err))
 
                 cursor.close()
 
@@ -184,7 +184,7 @@ class DB_Stats:
 				self.cnx.commit()
 
                 except mysql.connector.Error as err:
-                        print("Fail {}: {}".format(cursor.statement, err))
+                        print("Fail {0}: {1}".format(cursor.statement, err))
 			res = -1
 
                 cursor.close()
@@ -197,13 +197,13 @@ class DB_Stats:
                         cursor.execute(op)
 			result = cursor.fetchone()
 			if result != None:
-				print ("last record: {}".format(result[0]))
+				print ("last record: {0}".format(str(datetime.fromtimestamp (result[0]))))
                                 return result[0]
                         else:
                                 print ("Last date record is None")
 				return 0
                 except mysql.connector.Error as err:
-                        print("Fail {}: {}".format(cursor.statement, err))
+                        print("Fail {0}: {1}".format(cursor.statement, err))
                 cursor.close()
 
 
@@ -214,7 +214,7 @@ class DB_Stats:
 		    	self.cnx.database = self.db_name
 			cursor.close()
 		except mysql.connector.Error as err:
-			print("CONNECT FAIL {}".format (err))
+			print("CONNECT FAIL {0}".format (err))
 	
 	def most_activity_used (self, start, end):
 		uptime_last=0
@@ -237,7 +237,7 @@ class DB_Stats:
 						uptime_last= uptime[0]
 						activity_name = name[0]
                 except mysql.connector.Error as err:
-			print("Fail {}:".format(err))
+			print("Fail {0}:".format(err))
 		except:
                         print("most_activity_used Fail ")
 
@@ -255,7 +255,7 @@ class DB_Stats:
 			res = cursor.fetchone()
 			return res[0]
 		except mysql.connector.Error as err:
-                        print("Fail {}: {}".format(cursor.statement, err))
+                        print("Fail {0}: {1}".format(cursor.statement, err))
 		cursor.close()	
 
 	
diff --git a/sugar_stats_consolidation/rrd_files.py b/sugar_stats_consolidation/rrd_files.py
index 171dbe9..6c294f8 100644
--- a/sugar_stats_consolidation/rrd_files.py
+++ b/sugar_stats_consolidation/rrd_files.py
@@ -18,12 +18,14 @@ class RRD:
 			self.date_start = str(rrdtool.first(str(os.path.join (path,name))))
 		else:
 			self.date_start = str(date_start)
-			
 
 		if date_end == None:
 			self.date_end = str(rrdtool.last(str(os.path.join(path, name))))
 		else:
 			self.date_end   = str(date_end)
+
+		if self.date_start > self.date_end:
+			raise Exception("No data available on rrd, from {0} to {1}".format( str(datetime.fromtimestamp(float(self.date_start))), str(datetime.fromtimestamp(float(self.date_end)))))
 		
 		self.user_hash = os.path.split(path)[1]
 		
@@ -36,19 +38,20 @@ class RRD:
 					)
 	        """		
 		self.uuid = self.get_uuid_from_file(self.user_path) 
-		"""	
+		"""
+		self.uuid = "for_tesing"	
 
 		print "*******************************************"
-		print "                  RRD                      "
-		print "start: " + self.date_start
-		print "end: "  + self.date_end
+		print "     creating a RRD instance               "
+		print "start: " + str(datetime.fromtimestamp(float(self.date_start)))
+		print "end: "  + str(datetime.fromtimestamp(float(self.date_end)))
 		print "PATH: " + path
 		print "RRD NAME: " + name
 		print "\n"
 		try:
 			self.rrd = rrdtool.fetch (str(os.path.join(path,name)), 'AVERAGE', '-r 60', '-s '+ self.date_start, '-e '+self.date_end)
 		except:
-			raise Exception("rrdtool.fetch FAIL")
+			raise Exception("rrdtool.fetch FAIL, from {0} to {1}".format( str(datetime.fromtimestamp(float(self.date_start))), str(datetime.fromtimestamp(float(self.date_end)))))
 
 		print "                   DS                       "			
 		for item in self.DS.keys():
diff --git a/sugar_stats_consolidation/test_cons.py b/sugar_stats_consolidation/test_cons.py
index 00f43b5..f58f9fc 100644..100755
--- a/sugar_stats_consolidation/test_cons.py
+++ b/sugar_stats_consolidation/test_cons.py
@@ -1,9 +1,13 @@
-import stats_consolidation
-from stats_consolidation import *
+#!/usr/bin/env python
+
+import sugar_stats_consolidation
+from sugar_stats_consolidation.db import *
+from sugar_stats_consolidation.rrd_files import *
+from sugar_stats_consolidation.consolidation import *
 
 db = DB_Stats('statistics', 'root', 'gustavo')
 db.create();
 
-con = Consolidation('/home/gustavo/AC/server_stats/sugar-stats/rrd', db)
+con = Consolidation('/var/lib/sugar-stats/rrd', db)
 
 con.process_rrds()