Get ready to pull april data

.gitignore

@@ -111,3 +111,6 @@ tags
 # postgresql
 
 database.ini
+
+# pickle
+*.pkl

py/clustering.py

@@ -1,4 +1,4 @@
-from util import getQuery
+from util import getQuery, pickleQuery
 import pandas as p
 import matplotlib.pyplot as plt
 import seaborn as sns
@@ -12,23 +12,57 @@ import seaborn as sns
 # 
 # qparams = ['%%1117', '20/04/2017']
 
+#query = """
+#SELECT read_date, period, AVG(kwh_tot) AS average
+#FROM public.coup_tall_april
+#GROUP BY read_date, period
+#ORDER BY read_date, period;
+#"""
+#
+#qparams = []
+#
+#df = getQuery(query, qparams)
+#
+#print(df.info())
+#
+#sns.set()
+#
+##sns.lineplot(x = 'read_time', y = 'kwh_tot', hue = 'icp_id', data = df)
+#sns.lineplot(x = 'period', y = 'average', hue = 'read_date', data = df)
+#
+#plt.show()
+
 query = """
-SELECT read_date, period, AVG(kwh_tot) AS average
-FROM public.coup_tall_april
-GROUP BY read_date, period
-ORDER BY read_date, period;
+SELECT comb.icp_id, comb.read_time, COALESCE(kwh_tot, 0) AS kwh_tot
+FROM
+(
+    SELECT read_time, icp_id
+    FROM
+    (
+        SELECT read_time 
+        FROM GENERATE_SERIES('2017-04-01 00:30:00'::timestamp, '2017-05-01 00:00:00'::timestamp, 
+            '30 minutes'::interval) read_time
+    ) AS tsdata CROSS JOIN
+    (
+        SELECT *
+        FROM
+        (
+            SELECT icp_id, COUNT(DISTINCT read_date) AS data_days 
+            FROM coup_prd.coupdatamaster
+            WHERE read_date >= to_date('01/01/2017','dd/mm/yyyy')
+                AND read_date <  to_date('01/01/2018','dd/mm/yyyy')
+                AND content_code = 'UN'
+            GROUP BY icp_id
+        ) AS cir 
+        WHERE data_days >= 360
+    ) AS qual_icp
+) AS comb
+LEFT JOIN
+(
+    SELECT *, read_date + CONCAT(period / 2, ':', period % 2 * 30, ':00')::time AS read_time
+    FROM public.coup_tall_april
+) AS tall_timestamp 
+ON comb.read_time = tall_timestamp.read_time AND comb.icp_id = tall_timestamp.icp_id;
 """
 
-qparams = []
-
-df = getQuery(query, qparams)
-
-print(df.info())
-
-sns.set()
-
-#sns.lineplot(x = 'read_time', y = 'kwh_tot', hue = 'icp_id', data = df)
-sns.lineplot(x = 'period', y = 'average', hue = 'read_date', data = df)
-
-plt.show()
-
+pickleQuery(query, "April.pkl")

py/util.py

@@ -2,13 +2,15 @@ import psycopg2 as pg
 from configparser import ConfigParser
 import pandas.io.sql as psql
 
+
 def config(filename='database.ini', section='postgresql'):
-    """Config parser from http://www.postgresqltutorial.com/postgresql-python/connect/"""
+    """Config parser from
+    http://www.postgresqltutorial.com/postgresql-python/connect/"""
     # create a parser
     parser = ConfigParser()
     # read config file
     parser.read(filename)
- 
+
     # get section, default to postgresql
     db = {}
     if parser.has_section(section):
@@ -16,11 +18,13 @@ def config(filename='database.ini', section='postgresql'):
         for param in params:
             db[param[0]] = param[1]
     else:
-        raise Exception('Section {0} not found in the {1} file'.format(section, filename))
- 
+        raise Exception(
+            'Section {0} not found in the {1} file'.format(section, filename))
+
     return db
 
-def getQuery(query, qparams = []):
+
+def getQuery(query, qparams=[]):
     """
     Get single query
     """
@@ -35,10 +39,12 @@ def getQuery(query, qparams = []):
         cur = conn.cursor()
 
         # Get table
-        dataframe = psql.read_sql(query, conn, params = qparams)
+        dataframe = psql.read_sql(query, conn, params=qparams)
 
         cur.close()
 
+        print("Table recieved")
+
         return dataframe
 
     except (Exception, pg.DatabaseError) as error:
@@ -50,6 +56,13 @@ def getQuery(query, qparams = []):
             conn.close()
             print('Database connection closed')
 
+
+def pickleQuery(query, path, qparams=[]):
+    dq = getQuery(query, qparams)
+    dq.to_pickle(path)
+    print("Table pickled")
+
+
 if __name__ == "__main__":
     dv = getQuery('SELECT version()').version[0]
     print('PostgreSQL database version:')