More sql and python

py/clustering.py

 import seaborn as sns
 
 
+# query = """
+# SELECT *, read_date + CONCAT(period / 2, ':', period %% 2 * 30, ':00')::time AS read_time
+# FROM public.coup_tall_april WHERE icp_id LIKE (%s) AND read_date = to_date(%s, 'dd/mm/yyyy') 
+# ORDER BY icp_id, read_time;
+# """
+# 
+# qparams = ['%%1117', '20/04/2017']
+
 query = """
-SELECT icp_id, read_date + CONCAT(period / 2, ':', period %% 2 * 30, ':00')::time AS read_time,
-    kwh_tot
-FROM public.coup_tall_april WHERE icp_id LIKE (%s) ORDER BY icp_id, read_time;
+SELECT read_date, period, AVG(kwh_tot) AS average
+FROM public.coup_tall_april
+GROUP BY read_date, period
+ORDER BY read_date, period;
 """
 
-qparams = ['%%1117']
+qparams = []
 
 df = getQuery(query, qparams)
 
 
 sns.set()
 
-sns.scatterplot(x = 'read_time', y = 'kwh_tot', hue = 'icp_id', style = 'icp_id', data = df)
+#sns.lineplot(x = 'read_time', y = 'kwh_tot', hue = 'icp_id', data = df)
+sns.lineplot(x = 'period', y = 'average', hue = 'read_date', data = df)
 
 plt.show()
+

sql/queries.pgsql

 -- Possible subset for further testing
 SELECT *, read_date + CONCAT(period / 2, ':', period % 2 * 30, ':00')::time AS read_time
 FROM public.coup_tall_april WHERE icp_id LIKE '%1117' ORDER BY icp_id, read_time;
+
+-- Calculate averages for each half-hour
+SELECT read_date, period, AVG(kwh_tot) AS average
+FROM public.coup_tall_april 
+GROUP BY read_date, period
+ORDER BY read_date, period;
+
+-- Gerate timestamp list
+SELECT dd FROM GENERATE_SERIES('2017-04-01'::timestamp, '2017-04-02'::timestamp, '30 minutes'::interval) dd;
+
+-- Fraction of icp's in april with 1440 entries
+SELECT SUM(CASE WHEN isum.c = 1440 THEN 1 ELSE 0 END)::numeric / COUNT(*)::numeric As frac FROM
+(
+    SELECT COUNT(*) AS c, icp_id FROM public.coup_tall_april 
+    GROUP BY icp_id
+) AS isum;
+
+-- Get numbers of ICPs present over whole dataset
+SELECT COUNT(*) as c, icp_id, read_date FROM coup_prd.coupdatamaster
+    WHERE read_date >= to_date('01/01/2017','dd/mm/yyyy')
+    AND read_date <  to_date('01/01/2018','dd/mm/yyyy')
+    AND content_code = 'UN'
+GROUP BY icp_id, read_date ORDER BY c DESC, read_date LIMIT 40;
+
+
+SELECT MIN(read_date) as min_date, MAX(read_date) as max_date FROM coup_prd.coupdatamaster;
+
+SELECT SUM(CASE WHEN cir.c > 365 THEN 1 ELSE 0 END), COUNT(*)
+FROM
+(
+    SELECT COUNT(*) as c, icp_id FROM coup_prd.coupdatamaster
+        WHERE read_date >= to_date('01/01/2017','dd/mm/yyyy')
+        AND read_date <  to_date('01/01/2018','dd/mm/yyyy')
+        AND content_code = 'UN'
+    GROUP BY icp_id ORDER BY c DESC
+) AS cir;