add consolidation.py

change creation of tables, remove ids of Users and Resources

7 files changed, 67 insertions, 24 deletions

diff --git a/src/consolidation.py b/src/consolidation.py
new file mode 100644
index 0000000..a804cb3
--- /dev/null
+++ b/src/consolidation.py
@@ -0,0 +1,22 @@
+import os
+import rrd_files
+import db
+
+from rrd_files import *
+from db import *
+
+class Consolidation:
+	
+	def __init__(self, path, db):
+		self.base_path = path
+		self.date_start = db.get_date_last_record()
+	
+	def process_rrds (self):
+		for id_hash in os.listdir(unicode(self.base_path)) :
+			for user_hash in os.listdir( unicode( os.path.join(self.base_path, id_hash) ) ):
+				print (unicode(os.path.join(self.base_path, id_hash, user_hash)))
+				for rrd in os.listdir( unicode(os.path.join(self.base_path, id_hash, user_hash)) ): 
+					rrd_path = unicode (os.path.join(self.base_path, id_hash, user_hash) )
+					print ("PATH: "+ rrd_path + "file: "+ rrd)
+					rrd = RRD (path=rrd_path, name=rrd, date_start=self.date_start, date_end=None)
+			
diff --git a/src/db.py b/src/db.py
index b47348d..4ff6a4a 100644
--- a/src/db.py
+++ b/src/db.py
@@ -9,30 +9,30 @@ class DB_Stats:
 	TABLES['Usages'] = (
 		"CREATE TABLE `Usages` (" 
 		"	`ts` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP," 
-		"	`user_id` INTEGER NOT NULL," 
-		"	`resource_id` INTEGER NOT NULL," 
+		"	`user_hash` CHAR(40) NOT NULL," 
+		"	`resource_name` CHAR(80)," 
 		"	`start_date` TIMESTAMP NOT NULL," 
 		"	`data_type` CHAR (30) NOT NULL,"
 		"	`data` INTEGER NOT NULL,"
-		"	PRIMARY KEY (`start_date`,`resource_id`, `data_type`)"
+		"	PRIMARY KEY (`start_date`,`resource_name`, `data_type`)"
 		"	)")
 
 	TABLES['Resources'] = (
 		"CREATE TABLE Resources ("
-		"	`id` INTEGER auto_increment unique,"	
 		"	`name` CHAR(250),"
 		"	PRIMARY KEY (name)"
 		"	)")
 
 	TABLES['Users'] = (
 		"CREATE TABLE Users("
-		"	`id` INTEGER auto_increment unique,"	
+                "       `hash` CHAR (40) NOT NULL,"	
+		"       `uuid` CHAR (32) NOT NULL,"
 		"	`machine_sn` CHAR(80),"
 		"	`age` INTEGER NOT NULL,"
 		"	`school` CHAR(80),"
 		"	`sw_version` CHAR (80),"
 		"	`ts` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,"
-		"	PRIMARY KEY (machine_sn)"
+		"	PRIMARY KEY (hash)"
 		"	)")
 
 	TABLES['Runs'] = (
@@ -96,20 +96,24 @@ class DB_Stats:
 
 
 	def store_activity_uptime(self, rrd, data):
+		""" 
+		FIXME: improving, saving all data int rrd object 
+		"""
 		cursor = self.cnx.cursor()
 		insert = ("INSERT INTO Usages " 
-				"(user_id, "
-				"resource_id, "
+				"(user_hash, "
+				"resource_name, "
 				"start_date, "
 				"data_type, "
 				"data) "
 				"VALUES (%s, %s, %s, %s ,%s) ")
-		print ("store_activity_uptime: {}".format(self.get_resource_id(data['resource_name'])))
-		info = ('none', self.get_resource_id(data['resource_name']), datetime.fromtimestamp(float(data['date_start'])), 'uptime', data['data'])	
+
+
+		info = ('none', data['resource_name'], datetime.fromtimestamp(float(data['date_start'])), 'uptime', data['data'])	
 		
 		try:
 			cursor.execute(insert, info)
-			if self.update_last_record(rrd.get_last_record()) == 0:	
+			if self.update_last_record(rrd.get_date_last_record()) == 0:	
 				self.cnx.commit()
 
 		except mysql.connector.Error as err:
@@ -119,7 +123,7 @@ class DB_Stats:
 
 	def get_resource_id(self, resource_name):
 		cursor = self.cnx.cursor()
-		op = ("SELECT id FROM Resources WHERE name = %s")
+		op = ("SELECT name FROM Resources WHERE name = %s")
 		params = (resource_name,)
 		try:	
 			cursor.execute(op, params)
@@ -168,7 +172,7 @@ class DB_Stats:
 
                 cursor.close()
 		return res
-	def get_last_record (self):
+	def get_date_last_record (self):
 		cursor = self.cnx.cursor()
                 op = ("SELECT UNIX_TIMESTAMP ((SELECT last_ts FROM Runs))")
                 try:
diff --git a/src/db.pyc b/src/db.pyc
index a28a9b8..d86e765 100644
--- a/src/db.pyc
+++ b/src/db.pyc
diff --git a/src/rrd_files.py b/src/rrd_files.py
index a9f9892..51c450a 100644
--- a/src/rrd_files.py
+++ b/src/rrd_files.py
@@ -1,4 +1,5 @@
 import rrdtool
+import os
 
 
 class RRD:
@@ -8,25 +9,28 @@ class RRD:
 	data_item = 2
 	DS = {'active':0, 'buddies':0, 'instances':0, 'new':0, 'resumed':0, 'uptime':0}
 
-	def __init__(self, rrd_path, rrd_name, date_start, date_end):
+	def __init__(self, path, name, date_start, date_end):
 
-		self.rrd_name = rrd_name
+		self.rrd_name = name
 
 		if date_start == None:
-			self.date_start = str(rrdtool.first(rrd_path +"/"+ rrd_name))
+			self.date_start = str(rrdtool.first(str(os.path.join (path,name))))
 		else:
 			self.date_start = str(date_start)
 			
 
-		if date_end == 0:
-			self.date_end = str(rrdtool.last(rrd_path +"/"+ rrd_name))
+		if date_end == None:
+			self.date_end = str(rrdtool.last(str(os.path.join(path,name))))
 		else:
 			self.date_end   = str(date_end)
-
+		print "******************************************"
 		print "start: " + self.date_start
-		print "end: " + self.date_end
-						
-		self.rrd = rrdtool.fetch (rrd_path +"/"+ rrd_name, 'AVERAGE', '-r 60', '-s '+ self.date_start, '-e '+self.date_end)
+		print "end: "  + self.date_end
+		print "PATH: " + path
+		print "RRD NAME: " + name
+		print "******************************************"
+					
+		self.rrd = rrdtool.fetch (str(os.path.join(path,name)), 'AVERAGE', '-r 60', '-s '+ self.date_start, '-e '+self.date_end)
 
 		for item in self.DS.keys():
 			idx = self.get_ds_index (item)
@@ -87,3 +91,7 @@ class RRD:
 
 	def get_last_record(self):
 		return self.date_end
+
+	def set_user_hash(self, u_hash):
+		self.user_hash = u_hash
+
diff --git a/src/rrd_files.pyc b/src/rrd_files.pyc
index c191527..7c34e78 100644
--- a/src/rrd_files.pyc
+++ b/src/rrd_files.pyc
diff --git a/src/test_cons.py b/src/test_cons.py
new file mode 100644
index 0000000..c91ed42
--- /dev/null
+++ b/src/test_cons.py
@@ -0,0 +1,9 @@
+import consolidation
+from consolidation import *
+
+db = DB_Stats('statistics', 'root', 'gustavo')
+db.create();
+
+con = Consolidation('/home/gustavo/AC/server_stats/sugar-stats/rrd', db)
+
+con.process_rrds()
diff --git a/src/test_rrd.py b/src/test_rrd.py
index fe2aaf3..13b1e4b 100644
--- a/src/test_rrd.py
+++ b/src/test_rrd.py
@@ -24,10 +24,10 @@ DATE_END = datetime(year=2012,
                                 minute=0, 
                                 second=0).strftime("%s")
 
-DATE_START = db.get_last_record()
+DATE_START = db.get_date_last_record()
 DATE_END = datetime.now().strftime("%s")
 
-act_rrd = RRD (rrd_path = "/home/gustavo/AC/xo_stats", rrd_name="pippy.rrd", date_start=DATE_START, date_end=DATE_END)
+act_rrd = RRD (path = "/home/gustavo/AC/xo_stats", name="pippy.rrd", date_start=DATE_START, date_end=DATE_END)
 """
 act_rrd.show_valid_ds("uptime")
 act_rrd.show_valid_ds("resumed")