Data back to April 2016

py/clustering.py

@@ -36,7 +36,7 @@ from scipy.cluster.hierarchy import dendrogram, linkage, cophenet, fcluster
 # plt.show()
 
 numclusts = 9
-df = p.read_pickle('../data/2017-sample.pkl')
+df = p.read_pickle('../data/2016-17-sample.pkl')
 dforig = df
 
 print(df.info())
@@ -81,36 +81,36 @@ print(mdagg.info())
 print(mdagg.describe())
 # mdf.to_csv('~/windows/Documents/clusters-ward.csv')
 print("Saving")
-mdf.to_pickle('../data/9-clusters.pkl')
-mdagg.to_pickle('../data/9-clusters.agg.pkl')
+mdf.to_pickle('../data/9-clusters-1617.pkl')
+mdagg.to_pickle('../data/9-clusters-1617.agg.pkl')
 print("saved")
 
 # Algorithm via 
 # <https://stackoverflow.com/questions/38153829/custom-cluster-colors-of-scipy-dendrogram-in-python-link-color-func>
-# ldict = {icp_id:cpal[cluster] for icp_id, cluster in zip(clustdf.icp_id, clustdf.cluster)}
-# link_cols = {}
-# for i, i12 in enumerate(lobj[:,:2].astype(int)):
-#   c1, c2 = (link_cols[x] if x > len(lobj) else ldict[clustdf.icp_id[x]]
-#     for x in i12)
-#   link_cols[i+1+len(lobj)] = c1 if c1 == c2 else '#000000'
-# 
-# plt.figure(figsize = (25, 10))
-# plt.title('ICP Clustering Dendrogram')
-# plt.xlabel('ICP ID/(Number of ICPs)')
-# plt.ylabel('distance')
-# dendrogram(
-#     lobj,
-#     labels = cmat.index.values,
-#     leaf_rotation=90,
-#     leaf_font_size=8,
-#     # show_leaf_counts = True,
-#     # truncate_mode = 'lastp',
-#     # p = 50,
-#     # show_contracted = True,
-#     link_color_func = lambda x: link_cols[x],
-#     color_threshold = None
-# )
-# plt.show()
+ldict = {icp_id:cpal[cluster] for icp_id, cluster in zip(clustdf.icp_id, clustdf.cluster)}
+link_cols = {}
+for i, i12 in enumerate(lobj[:,:2].astype(int)):
+  c1, c2 = (link_cols[x] if x > len(lobj) else ldict[clustdf.icp_id[x]]
+    for x in i12)
+  link_cols[i+1+len(lobj)] = c1 if c1 == c2 else '#000000'
+
+plt.figure(figsize = (25, 10))
+plt.title('ICP Clustering Dendrogram')
+plt.xlabel('ICP ID/(Number of ICPs)')
+plt.ylabel('distance')
+dendrogram(
+    lobj,
+    labels = cmat.index.values,
+    leaf_rotation=90,
+    leaf_font_size=8,
+    # show_leaf_counts = True,
+    # truncate_mode = 'lastp',
+    # p = 50,
+    # show_contracted = True,
+    link_color_func = lambda x: link_cols[x],
+    color_threshold = None
+)
+plt.show()
 
 # sns.set()
 # 

py/downkwh.py

@@ -39,9 +39,9 @@ import seaborn as sns
 # 
 # pickleQuery(query, "../data/jan19s.pkl")
 
-kwhdata = getkwh('2017-01-01', '2018-01-01', '2017-01-01 00:30:00', '2018-01-01 00:00:00', '%%1')
+kwhdata = getkwh('2016-04-01', '2018-01-01', '2016-04-01 00:30:00', '2018-01-01 00:00:00', '%%1')
 
 print(kwhdata.info())
 
 print("Pickling")
-kwhdata.to_pickle("../data/2017-sample.pkl")
+kwhdata.to_pickle("../data/2016-17-sample.pkl")

py/util.py

@@ -3,6 +3,7 @@ from configparser import ConfigParser
 import pandas.io.sql as psql
 import datetime as dt
 import numpy as np
+from pprint import pprint
 
 
 def config(filename='database.ini', section='postgresql'):
@@ -110,7 +111,7 @@ def getkwh(datestart, dateend, timestart, timeend, subset):
         # 'subset': subset
     }
     print("Getting data with parameters:")
-    print(pdict)
+    pprint(pdict)
     qdf = getQuery(query, pdict)
     print("Optimising")
     qdf['icp_id'] = qdf['icp_id'].astype(np.int32)

sql/queries.pgsql

@@ -364,3 +364,7 @@ CREATE TABLE public.icp_sample AS
 SELECT * FROM public.best_icp
 ORDER BY random()
 LIMIT 1000;
+
+-- range of date values
+SELECT MIN(read_date) AS mindate, MAX(read_date) AS maxdate
+FROM coup_prd.coupdatamaster;