Download and collate in wide form, month-by-month

py/collate.py

@@ -0,0 +1,19 @@
+# Collate 12 dataframes into one (wide) combined dataframe
+import pandas as p
+import gc
+
+
+months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
+
+coldf = p.read_pickle('../data/2017-{}-all.pkl'.format(months[0]))
+
+for i in range(1, 12):
+    tdf = p.read_pickle('../data/2017-{}-all.pkl'.format(months[i])) 
+    coldf = p.concat([coldf, tdf])
+    del tdf
+    gc.collect()
+
+print(coldf.info())
+
+coldf.to_pickle('../data/2017-all-wide.pkl')
+

py/downkwh.py

@@ -7,19 +7,19 @@ months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct',
 mstarts = list(range(1, 13))
 mends = mstarts[1:13]
 mends.append(1)
-
-print(months)
-print(mstarts)
-print(mends)
+yends = [2017] * 11
+yends.append(2018)
 
 for i, m in enumerate(months):
+    if i < 11:
+        continue
     print(i)
     print(m)
     print(datetime.now().time())
     kwhdata = getkwh('2017-{:02d}-01'.format(mstarts[i]), 
-                     '2017-{:02d}-01'.format(mends[i]), 
+                     '{}-{:02d}-01'.format(yends[i], mends[i]), 
                      '2017-{:02d}-01 00:30:00'.format(mstarts[i]), 
-                     '2017-{:02d}-01 00:00:00'.format(mends[i]), 
+                     '{}-{:02d}-01 00:00:00'.format(yends[i], mends[i]), 
                      '%%1')
     print("Pivoting")
     kwhpiv = kwhdata.pivot(index = 'read_time', columns = 'icp_id', values = 'kwh_tot')