July and January stuff

py/clustering.py

@@ -34,7 +34,7 @@ from scipy.cluster.hierarchy import dendrogram, linkage, cophenet, fcluster
 # 
 # plt.show()
 
-df = p.read_pickle('../data/April17s.pkl')
+df = p.read_pickle('../data/jan19s.pkl')
 dforig = df
 
 print(df.info())
@@ -53,36 +53,36 @@ lobj = linkage(lmat, method = 'ward')
 print(lobj)
 print(cophenet(lobj, lmat))
 
-plt.figure(figsize = (25, 10))
-plt.title('ICP Clustering Dendrogram')
-plt.xlabel('ICP ID/(Number of ICPs)')
-plt.ylabel('distance')
-dendrogram(
-    lobj,
-    labels = cmat.index.values,
-    leaf_rotation=90,
-    leaf_font_size=8,
-    #show_leaf_counts = True,
-    #truncate_mode = 'lastp',
-    #p = 50,
-    #show_contracted = True,
-    color_threshold = 2.1
-)
-plt.show()
-
-#clusts = fcluster(lobj, 5, criterion='maxclust')
-#print(clusts)
-#print(cmat.index.values)
-#clustdf = p.DataFrame({'icp_id' : cmat.index.values, 'cluster' : [chr(x + ord('A') - 1) for x in clusts]})
-#print(clustdf)
-#mdf = p.merge(clustdf, dforig, on = 'icp_id', how = 'left')
-#print(mdf)
-#print(mdf.info())
-#print(mdf.cluster.describe())
-#
-#mdf.to_csv('~/windows/Documents/clusters-ward.csv')
-#
-#sns.set()
-#
-#sns.lineplot(x = 'read_time', y = 'kwh_tot', hue = 'cluster', data = mdf)
+#plt.figure(figsize = (25, 10))
+#plt.title('ICP Clustering Dendrogram')
+#plt.xlabel('ICP ID/(Number of ICPs)')
+#plt.ylabel('distance')
+#dendrogram(
+#    lobj,
+#    labels = cmat.index.values,
+#    leaf_rotation=90,
+#    leaf_font_size=8,
+#    #show_leaf_counts = True,
+#    #truncate_mode = 'lastp',
+#    #p = 50,
+#    #show_contracted = True,
+#    color_threshold = 1.9
+#)
 #plt.show()
+
+clusts = fcluster(lobj, 6, criterion='maxclust')
+print(clusts)
+print(cmat.index.values)
+clustdf = p.DataFrame({'icp_id' : cmat.index.values, 'cluster' : [chr(x + ord('A') - 1) for x in clusts]})
+print(clustdf)
+mdf = p.merge(clustdf, dforig, on = 'icp_id', how = 'left')
+print(mdf)
+print(mdf.info())
+print(mdf.cluster.describe())
+
+mdf.to_csv('~/windows/Documents/clusters-ward.csv')
+
+sns.set()
+
+sns.lineplot(x = 'read_time', y = 'kwh_tot', hue = 'cluster', data = mdf)
+plt.show()

py/downkwh.py

@@ -11,7 +11,7 @@ FROM
     FROM
     (
         SELECT read_time 
-        FROM GENERATE_SERIES('2017-04-01 00:30:00'::timestamp, '2017-05-01 00:00:00'::timestamp, 
+        FROM GENERATE_SERIES('2017-01-01 00:30:00'::timestamp, '2017-02-01 00:00:00'::timestamp, 
             '30 minutes'::interval) read_time
     ) AS tsdata CROSS JOIN
     (
@@ -23,7 +23,7 @@ FROM
             WHERE read_date >= to_date('01/01/2017','dd/mm/yyyy')
                 AND read_date <  to_date('01/01/2018','dd/mm/yyyy')
                 AND content_code = 'UN'
-                AND icp_id LIKE '%%17'
+                AND icp_id LIKE '%%19'
             GROUP BY icp_id
         ) AS cir 
         WHERE data_days >= 360
@@ -32,9 +32,9 @@ FROM
 LEFT JOIN
 (
     SELECT *, read_date + CONCAT(period / 2, ':', period %% 2 * 30, ':00')::time AS read_time
-    FROM public.coup_tall_april
+    FROM public.coup_tall_jan
 ) AS tall_timestamp 
 ON comb.read_time = tall_timestamp.read_time AND comb.icp_id = tall_timestamp.icp_id;
 """
 
-pickleQuery(query, "../data/April17s.pkl")
+pickleQuery(query, "../data/jan19s.pkl")

sql/queries.pgsql

@@ -262,3 +262,39 @@ LEFT JOIN
 ) AS tall_timestamp 
 ON comb.read_time = tall_timestamp.read_time AND comb.icp_id = tall_timestamp.icp_id
 GROUP BY comb.icp_id;
+
+-- Move to July
+
+-- Create "tall" table for July 2017 (mod from above) 
+CREATE TABLE public.coup_tall_july AS
+SELECT  a.icp_id
+     , a.read_date
+     , c.period
+     , sum(c.read_kwh) as kwh_tot
+     , sum(case when a.content_code = 'UN' then c.read_kwh else 0 end) as kwh_un
+     , sum(case when a.content_code in ('CN','EG') then c.read_kwh else 0 end) as kwh_cn
+FROM    coup_prd.coupdatamaster a,
+	unnest(a.read_array) WITH ORDINALITY c(read_kwh, period)
+WHERE   a.read_date >= to_date('01/07/2017','dd/mm/yyyy')
+ and   a.read_date <  to_date('01/08/2017','dd/mm/yyyy')
+ and   a.content_code  ~ ('UN|CN|EG')
+GROUP BY 1, 2, 3
+ORDER BY 1, 2, 3;
+
+-- Move to January
+
+-- Create "tall" table for Jan 2017 (mod from above) 
+CREATE TABLE public.coup_tall_jan AS
+SELECT  a.icp_id
+     , a.read_date
+     , c.period
+     , sum(c.read_kwh) as kwh_tot
+     , sum(case when a.content_code = 'UN' then c.read_kwh else 0 end) as kwh_un
+     , sum(case when a.content_code in ('CN','EG') then c.read_kwh else 0 end) as kwh_cn
+FROM    coup_prd.coupdatamaster a,
+	unnest(a.read_array) WITH ORDINALITY c(read_kwh, period)
+WHERE   a.read_date >= to_date('01/01/2017','dd/mm/yyyy')
+ and   a.read_date <  to_date('01/02/2017','dd/mm/yyyy')
+ and   a.content_code  ~ ('UN|CN|EG')
+GROUP BY 1, 2, 3
+ORDER BY 1, 2, 3;