From 56395972aa593bc221c1689c703815781185ddcb Mon Sep 17 00:00:00 2001 From: Gustavo Duarte Date: Sat, 22 Dec 2012 18:46:02 +0000 Subject: fixing parsing rrds directory layout, on consolidation.pywqxxxx --- diff --git a/src/consolidation.py b/src/consolidation.py index a804cb3..23f4fce 100644 --- a/src/consolidation.py +++ b/src/consolidation.py @@ -10,13 +10,24 @@ class Consolidation: def __init__(self, path, db): self.base_path = path self.date_start = db.get_date_last_record() - + self.db = db def process_rrds (self): - for id_hash in os.listdir(unicode(self.base_path)) : - for user_hash in os.listdir( unicode( os.path.join(self.base_path, id_hash) ) ): - print (unicode(os.path.join(self.base_path, id_hash, user_hash))) - for rrd in os.listdir( unicode(os.path.join(self.base_path, id_hash, user_hash)) ): - rrd_path = unicode (os.path.join(self.base_path, id_hash, user_hash) ) - print ("PATH: "+ rrd_path + "file: "+ rrd) - rrd = RRD (path=rrd_path, name=rrd, date_start=self.date_start, date_end=None) - + id_hash_list = os.listdir(unicode(self.base_path)) + if id_hash_list: + for id_hash in id_hash_list: + user_hash_list = os.listdir( unicode( os.path.join(self.base_path, id_hash) ) ) + if user_hash_list: + for user_hash in user_hash_list: + rrd_list = os.listdir( unicode(os.path.join(self.base_path, id_hash, user_hash)) ) + if rrd_list: + for rrd in rrd_list: + rrd_path = unicode (os.path.join(self.base_path, id_hash, user_hash) ) + rrd_obj = RRD (path=rrd_path, name=rrd, date_start=self.date_start, date_end=None) + self.db.store_activity_uptime(rrd_obj) + else: + print "None rrd file found" + os.path.join(self.base_path, id_hash, user_hash) + else: + print "None hash user found on: " + os.path.join(self.base_path, id_hash) + else: + print "None hash ids found on: " + self.base_path + diff --git a/src/db.py b/src/db.py index 4ff6a4a..889577b 100644 --- a/src/db.py +++ b/src/db.py @@ -95,10 +95,10 @@ class DB_Stats: - def store_activity_uptime(self, rrd, data): - """ - FIXME: improving, saving all data int rrd object - """ + def store_activity_uptime(self, rrd): + + self.store_resource(rrd.get_name()) + cursor = self.cnx.cursor() insert = ("INSERT INTO Usages " "(user_hash, " @@ -108,20 +108,19 @@ class DB_Stats: "data) " "VALUES (%s, %s, %s, %s ,%s) ") + for d in rrd.get_uptime_by_interval(): + info = ('none', rrd.get_name() , datetime.fromtimestamp(float(d[0])), 'uptime', d[1]) + try: + cursor.execute(insert, info) + if self.update_last_record(rrd.get_date_last_record()) == 0: + self.cnx.commit() - info = ('none', data['resource_name'], datetime.fromtimestamp(float(data['date_start'])), 'uptime', data['data']) - - try: - cursor.execute(insert, info) - if self.update_last_record(rrd.get_date_last_record()) == 0: - self.cnx.commit() - - except mysql.connector.Error as err: - print("Fail INSERT: {}".format(err)) + except mysql.connector.Error as err: + print("Fail INSERT: {}".format(err)) cursor.close() - def get_resource_id(self, resource_name): + def store_resource(self, resource_name): cursor = self.cnx.cursor() op = ("SELECT name FROM Resources WHERE name = %s") params = (resource_name,) @@ -129,16 +128,12 @@ class DB_Stats: cursor.execute(op, params) result = cursor.fetchone() if result != None: - print ("result is not None: {}".format(result)) - return result[0] + print("Resource {} already in db".format(resource_name)) else: - print ("result is None") insert = ("INSERT INTO Resources (name) VALUES (%s)") info = (resource_name, ) cursor.execute(insert, info) self.cnx.commit() - """ TODO: return ID instead call recursevily """ - self.get_resource_id(resource_name) except mysql.connector.Error as err: print("Fail SELCT: {}".format(err)) @@ -172,6 +167,7 @@ class DB_Stats: cursor.close() return res + def get_date_last_record (self): cursor = self.cnx.cursor() op = ("SELECT UNIX_TIMESTAMP ((SELECT last_ts FROM Runs))") @@ -179,7 +175,7 @@ class DB_Stats: cursor.execute(op) result = cursor.fetchone() if result != None: - print ("last rec: {}".format(result[0])) + print ("last record: {}".format(result[0])) return result[0] else: print ("result is None") diff --git a/src/db.pyc b/src/db.pyc index d86e765..1530861 100644 --- a/src/db.pyc +++ b/src/db.pyc Binary files differ diff --git a/src/rrd_files.py b/src/rrd_files.py index 51c450a..3fbf154 100644 --- a/src/rrd_files.py +++ b/src/rrd_files.py @@ -52,6 +52,7 @@ class RRD: def get_uptime_by_interval (self): ds_name = "uptime" res=list() + print "------------------- Calcule "+ ds_name +"---------------------" i=0 found = False @@ -89,7 +90,7 @@ class RRD: print "---------------------------------------------------" - def get_last_record(self): + def get_date_last_record(self): return self.date_end def set_user_hash(self, u_hash): diff --git a/src/rrd_files.pyc b/src/rrd_files.pyc index 7c34e78..316839b 100644 --- a/src/rrd_files.pyc +++ b/src/rrd_files.pyc Binary files differ diff --git a/src/test_rrd.py b/src/test_rrd.py index 13b1e4b..0fb2028 100644 --- a/src/test_rrd.py +++ b/src/test_rrd.py @@ -27,7 +27,7 @@ DATE_END = datetime(year=2012, DATE_START = db.get_date_last_record() DATE_END = datetime.now().strftime("%s") -act_rrd = RRD (path = "/home/gustavo/AC/xo_stats", name="pippy.rrd", date_start=DATE_START, date_end=DATE_END) +act_rrd = RRD (path = "/home/gustavo/AC/consolidation/rrds", name="pippy.rrd", date_start=DATE_START, date_end=DATE_END) """ act_rrd.show_valid_ds("uptime") act_rrd.show_valid_ds("resumed") @@ -36,9 +36,4 @@ act_rrd.show_valid_ds("instances") act_rrd.show_valid_ds("buddies") """ data = {} -for item in act_rrd.get_uptime_by_interval (): - data['date_start'] = item[0] - data['data'] = item[1] - data['resource_name'] = act_rrd.get_name() - """FIXME implement all data load inside of RRD object""" - db.store_activity_uptime(act_rrd, data) +db.store_activity_uptime(act_rrd) -- cgit v0.9.1