Optimisations

py/clustering.py

@@ -36,7 +36,7 @@ from scipy.cluster.hierarchy import dendrogram, linkage, cophenet, fcluster
 # plt.show()
 
 numclusts = 7
-df = p.read_pickle('../data/2017-20s.pkl')
+df = p.read_pickle('../data/2017-1s.pkl')
 dforig = df
 
 print(df.info())

py/downkwh.py

@@ -39,9 +39,9 @@ import seaborn as sns
 # 
 # pickleQuery(query, "../data/jan19s.pkl")
 
-kwhdata = getkwh('2017-01-01', '2018-01-01', '2017-01-01 00:30:00', '2018-01-01 00:00:00', '%%20')
+kwhdata = getkwh('2017-01-01', '2018-01-01', '2017-01-01 00:30:00', '2018-01-01 00:00:00', '%%1')
 
 print(kwhdata.info())
 
 print("Pickling")
-kwhdata.to_pickle("../data/2017-20s.pkl")
+kwhdata.to_pickle("../data/2017-1s.pkl")

py/util.py

@@ -2,6 +2,7 @@ import psycopg2 as pg
 from configparser import ConfigParser
 import pandas.io.sql as psql
 import datetime as dt
+import numpy as np
 
 
 def config(filename='database.ini', section='postgresql'):
@@ -67,7 +68,7 @@ def pickleQuery(query, path, qparams=[]):
 
 def getkwh(datestart, dateend, timestart, timeend, subset):
     query = """
-    SELECT comb.icp_id, comb.read_time, COALESCE(kwh_tot, 0) AS kwh_tot
+    SELECT SUBSTRING(comb.icp_id FROM 2 FOR 6)::int AS icp_id, comb.read_time, COALESCE(kwh_tot, 0) AS kwh_tot
     FROM
     (
         SELECT read_time, icp_id
@@ -123,6 +124,9 @@ def getkwh(datestart, dateend, timestart, timeend, subset):
     print("Getting data with parameters:")
     print(pdict)
     qdf = getQuery(query, pdict)
+    print("Optimising")
+    qdf['icp_id'] = qdf['icp_id'].astype(np.int32)
+    qdf['kwh_tot'] = qdf['kwh_tot'].astype(np.float32)
     print("Done")
     return(qdf)