Download only 5k sample

py/downkwh.py

@@ -2,6 +2,7 @@ from util import getQuery, pickleQuery, getkwh
 import pandas as p
 import gc
 from datetime import datetime
+from tqdm import tqdm
 
 months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
 mstarts = list(range(1, 13))
@@ -10,10 +11,9 @@ mends.append(1)
 yends = [2017] * 11
 yends.append(2018)
 
-for i, m in enumerate(months):
-    if i < 11:
-        continue
-    print(i)
+for i, m in tqdm(enumerate(months)):
+    # if i < 11:
+    #     continue
     print(m)
     print(datetime.now().time())
     kwhdata = getkwh('2017-{:02d}-01'.format(mstarts[i]), 
@@ -24,7 +24,7 @@ for i, m in enumerate(months):
     print("Pivoting")
     kwhpiv = kwhdata.pivot(index = 'read_time', columns = 'icp_id', values = 'kwh_tot')
     print("Pickling")
-    kwhpiv.to_pickle('../data/2017-{}-all.pkl'.format(m))
+    kwhpiv.to_pickle('../data/2017-{}-5k.pkl'.format(m))
     del kwhdata
     del kwhpiv
     gc.collect()

py/util.py

@@ -79,7 +79,7 @@ def getkwh(datestart, dateend, timestart, timeend, subset):
             SELECT read_time 
             FROM GENERATE_SERIES(%(tsstart)s::timestamp, %(tsend)s::timestamp, 
                 '30 minutes'::interval) read_time
-        ) AS tsdata CROSS JOIN public.best_icp
+        ) AS tsdata CROSS JOIN public.icp_sample_5k
     ) AS comb
     LEFT JOIN
     (
@@ -97,7 +97,7 @@ def getkwh(datestart, dateend, timestart, timeend, subset):
              and   a.read_date <  to_date(%(dateend)s,'yyyy-mm-dd')
              and   a.content_code  ~ ('UN|CN|EG')
              AND   a.icp_id IN (
-                SELECT icp_id FROM public.best_icp
+                SELECT icp_id FROM public.icp_sample_5k
              )
             GROUP BY 1, 2, 3
         ) AS coup_tall