fixing parsing rrds directory layout, on consolidation.pywqxxxx

6 files changed, 40 insertions, 37 deletions

diff --git a/src/consolidation.py b/src/consolidation.py
index a804cb3..23f4fce 100644
--- a/src/consolidation.py
+++ b/src/consolidation.py
@@ -10,13 +10,24 @@ class Consolidation:
 	def __init__(self, path, db):
 		self.base_path = path
 		self.date_start = db.get_date_last_record()
-	
+		self.db = db	
 	def process_rrds (self):
-		for id_hash in os.listdir(unicode(self.base_path)) :
-			for user_hash in os.listdir( unicode( os.path.join(self.base_path, id_hash) ) ):
-				print (unicode(os.path.join(self.base_path, id_hash, user_hash)))
-				for rrd in os.listdir( unicode(os.path.join(self.base_path, id_hash, user_hash)) ): 
-					rrd_path = unicode (os.path.join(self.base_path, id_hash, user_hash) )
-					print ("PATH: "+ rrd_path + "file: "+ rrd)
-					rrd = RRD (path=rrd_path, name=rrd, date_start=self.date_start, date_end=None)
-			
+		id_hash_list = os.listdir(unicode(self.base_path))
+		if id_hash_list:
+			for id_hash in id_hash_list:
+				user_hash_list = os.listdir( unicode( os.path.join(self.base_path, id_hash) ) )
+				if user_hash_list:
+					for user_hash in user_hash_list:
+						rrd_list = os.listdir( unicode(os.path.join(self.base_path, id_hash, user_hash)) )
+						if rrd_list:
+							for rrd in rrd_list: 
+								rrd_path = unicode (os.path.join(self.base_path, id_hash, user_hash) )
+								rrd_obj = RRD (path=rrd_path, name=rrd, date_start=self.date_start, date_end=None)
+								self.db.store_activity_uptime(rrd_obj)
+						else:
+							print "None rrd file found" + os.path.join(self.base_path, id_hash, user_hash)
+				else:
+					print "None hash user found on: " + os.path.join(self.base_path, id_hash)	
+		else:
+			print "None hash ids  found on: " + self.base_path
+						
diff --git a/src/db.py b/src/db.py
index 4ff6a4a..889577b 100644
--- a/src/db.py
+++ b/src/db.py
@@ -95,10 +95,10 @@ class DB_Stats:
 
 
 
-	def store_activity_uptime(self, rrd, data):
-		""" 
-		FIXME: improving, saving all data int rrd object 
-		"""
+	def store_activity_uptime(self, rrd):
+		
+		self.store_resource(rrd.get_name())
+	
 		cursor = self.cnx.cursor()
 		insert = ("INSERT INTO Usages " 
 				"(user_hash, "
@@ -108,20 +108,19 @@ class DB_Stats:
 				"data) "
 				"VALUES (%s, %s, %s, %s ,%s) ")
 
+		for d in rrd.get_uptime_by_interval():
+			info = ('none', rrd.get_name() , datetime.fromtimestamp(float(d[0])), 'uptime', d[1])	
+			try:
+				cursor.execute(insert, info)
+				if self.update_last_record(rrd.get_date_last_record()) == 0:	
+					self.cnx.commit()
 
-		info = ('none', data['resource_name'], datetime.fromtimestamp(float(data['date_start'])), 'uptime', data['data'])	
-		
-		try:
-			cursor.execute(insert, info)
-			if self.update_last_record(rrd.get_date_last_record()) == 0:	
-				self.cnx.commit()
-
-		except mysql.connector.Error as err:
-			print("Fail INSERT: {}".format(err))
+			except mysql.connector.Error as err:
+				print("Fail INSERT: {}".format(err))
 		
 		cursor.close()
 
-	def get_resource_id(self, resource_name):
+	def store_resource(self, resource_name):
 		cursor = self.cnx.cursor()
 		op = ("SELECT name FROM Resources WHERE name = %s")
 		params = (resource_name,)
@@ -129,16 +128,12 @@ class DB_Stats:
 			cursor.execute(op, params)
 			result = cursor.fetchone()
 			if result != None:
-				print ("result is not None: {}".format(result))
-				return result[0]
+				print("Resource {} already in db".format(resource_name))
 			else:
-				print ("result is None")
 				insert = ("INSERT INTO Resources (name) VALUES (%s)")
 				info = (resource_name, )
 				cursor.execute(insert, info)
 				self.cnx.commit()
-				""" TODO: return ID instead call recursevily """
-				self.get_resource_id(resource_name)
 		except mysql.connector.Error as err:
 			print("Fail SELCT: {}".format(err))
 
@@ -172,6 +167,7 @@ class DB_Stats:
 
                 cursor.close()
 		return res
+
 	def get_date_last_record (self):
 		cursor = self.cnx.cursor()
                 op = ("SELECT UNIX_TIMESTAMP ((SELECT last_ts FROM Runs))")
@@ -179,7 +175,7 @@ class DB_Stats:
                         cursor.execute(op)
 			result = cursor.fetchone()
 			if result != None:
-				print ("last rec: {}".format(result[0]))
+				print ("last record: {}".format(result[0]))
                                 return result[0]
                         else:
                                 print ("result is None")
diff --git a/src/db.pyc b/src/db.pyc
index d86e765..1530861 100644
--- a/src/db.pyc
+++ b/src/db.pyc
diff --git a/src/rrd_files.py b/src/rrd_files.py
index 51c450a..3fbf154 100644
--- a/src/rrd_files.py
+++ b/src/rrd_files.py
@@ -52,6 +52,7 @@ class RRD:
 	def get_uptime_by_interval (self):
 		ds_name = "uptime"
 		res=list()
+	
 		print "------------------- Calcule "+ ds_name +"---------------------"
 		i=0
 		found = False
@@ -89,7 +90,7 @@ class RRD:
 		print "---------------------------------------------------"
 
 
-	def get_last_record(self):
+	def get_date_last_record(self):
 		return self.date_end
 
 	def set_user_hash(self, u_hash):
diff --git a/src/rrd_files.pyc b/src/rrd_files.pyc
index 7c34e78..316839b 100644
--- a/src/rrd_files.pyc
+++ b/src/rrd_files.pyc
diff --git a/src/test_rrd.py b/src/test_rrd.py
index 13b1e4b..0fb2028 100644
--- a/src/test_rrd.py
+++ b/src/test_rrd.py
@@ -27,7 +27,7 @@ DATE_END = datetime(year=2012,
 DATE_START = db.get_date_last_record()
 DATE_END = datetime.now().strftime("%s")
 
-act_rrd = RRD (path = "/home/gustavo/AC/xo_stats", name="pippy.rrd", date_start=DATE_START, date_end=DATE_END)
+act_rrd = RRD (path = "/home/gustavo/AC/consolidation/rrds", name="pippy.rrd", date_start=DATE_START, date_end=DATE_END)
 """
 act_rrd.show_valid_ds("uptime")
 act_rrd.show_valid_ds("resumed")
@@ -36,9 +36,4 @@ act_rrd.show_valid_ds("instances")
 act_rrd.show_valid_ds("buddies")
 """
 data = {}
-for item in act_rrd.get_uptime_by_interval ():
-	data['date_start'] = item[0]
-	data['data'] = item[1]
-	data['resource_name'] = act_rrd.get_name()
-	"""FIXME implement all data load inside of RRD object""" 
-	db.store_activity_uptime(act_rrd, data)
+db.store_activity_uptime(act_rrd)