Further optimisations

py/downkwh.py

@@ -19,9 +19,12 @@ for i, m in enumerate(months):
                      '2017-{:02d}-01 00:30:00'.format(mstarts[i]), 
                      '2017-{:02d}-01 00:00:00'.format(mends[i]), 
                      '%%1')
+    print("Pivoting")
+    kwhpiv = kwhdata.pivot(index = 'read_time', columns = 'icp_id', values = 'kwh_tot')
     print("Pickling")
-    kwhdata.to_pickle('../data/2017-{}-all.pkl'.format(m))
+    kwhpiv.to_pickle('../data/2017-{}-all.pkl'.format(m))
     del kwhdata
+    del kwhpiv
     gc.collect()
 
 print('Done')

py/util.py

@@ -116,7 +116,7 @@ def getkwh(datestart, dateend, timestart, timeend, subset):
     qdf = getQuery(query, pdict)
     print("Optimising")
     qdf['icp_id'] = qdf['icp_id'].astype(np.int32)
-    qdf['kwh_tot'] = qdf['kwh_tot'].astype(np.float32)
+    qdf['kwh_tot'] = qdf['kwh_tot'].astype(np.float16)
     print("Done")
     return(qdf)