Optimisations

py/clustering.py

 # plt.show()
 
 numclusts = 7
-df = p.read_pickle('../data/2017-20s.pkl')
+df = p.read_pickle('../data/2017-1s.pkl')
 dforig = df
 
 print(df.info())

py/downkwh.py

 # 
 # pickleQuery(query, "../data/jan19s.pkl")
 
-kwhdata = getkwh('2017-01-01', '2018-01-01', '2017-01-01 00:30:00', '2018-01-01 00:00:00', '%%20')
+kwhdata = getkwh('2017-01-01', '2018-01-01', '2017-01-01 00:30:00', '2018-01-01 00:00:00', '%%1')
 
 print(kwhdata.info())
 
 print("Pickling")
-kwhdata.to_pickle("../data/2017-20s.pkl")
+kwhdata.to_pickle("../data/2017-1s.pkl")

py/util.py

 from configparser import ConfigParser
 import pandas.io.sql as psql
 import datetime as dt
+import numpy as np
 
 
 def config(filename='database.ini', section='postgresql'):
 
 def getkwh(datestart, dateend, timestart, timeend, subset):
     query = """
-    SELECT comb.icp_id, comb.read_time, COALESCE(kwh_tot, 0) AS kwh_tot
+    SELECT SUBSTRING(comb.icp_id FROM 2 FOR 6)::int AS icp_id, comb.read_time, COALESCE(kwh_tot, 0) AS kwh_tot
     FROM
     (
         SELECT read_time, icp_id
     print("Getting data with parameters:")
     print(pdict)
     qdf = getQuery(query, pdict)
+    print("Optimising")
+    qdf['icp_id'] = qdf['icp_id'].astype(np.int32)
+    qdf['kwh_tot'] = qdf['kwh_tot'].astype(np.float32)
     print("Done")
     return(qdf)