Futher optimisation

py/clustering.py

@@ -36,7 +36,7 @@ from scipy.cluster.hierarchy import dendrogram, linkage, cophenet, fcluster
 # plt.show()
 
 numclusts = 7
-df = p.read_pickle('../data/2017-1s.pkl')
+df = p.read_pickle('../data/2017-sample.pkl')
 dforig = df
 
 print(df.info())

py/downkwh.py

@@ -44,4 +44,4 @@ kwhdata = getkwh('2017-01-01', '2018-01-01', '2017-01-01 00:30:00', '2018-01-01
 print(kwhdata.info())
 
 print("Pickling")
-kwhdata.to_pickle("../data/2017-1s.pkl")
+kwhdata.to_pickle("../data/2017-sample.pkl")

py/util.py

@@ -77,21 +77,7 @@ def getkwh(datestart, dateend, timestart, timeend, subset):
             SELECT read_time 
             FROM GENERATE_SERIES(%(tsstart)s::timestamp, %(tsend)s::timestamp, 
                 '30 minutes'::interval) read_time
-        ) AS tsdata CROSS JOIN
-        (
-            SELECT *
-            FROM
-            (
-                SELECT icp_id, COUNT(DISTINCT read_date) AS data_days 
-                FROM coup_prd.coupdatamaster
-                WHERE read_date >= to_date('01/01/2017','dd/mm/yyyy')
-                    AND read_date <  to_date('01/01/2018','dd/mm/yyyy')
-                    AND content_code = 'UN'
-                    AND icp_id LIKE %(subset)s
-                GROUP BY icp_id
-            ) AS cir 
-            WHERE data_days >= 360
-        ) AS qual_icp
+        ) AS tsdata CROSS JOIN public.icp_sample
     ) AS comb
     LEFT JOIN
     (
@@ -108,7 +94,9 @@ def getkwh(datestart, dateend, timestart, timeend, subset):
             WHERE   a.read_date >= to_date(%(datestart)s,'yyyy-mm-dd')
              and   a.read_date <  to_date(%(dateend)s,'yyyy-mm-dd')
              and   a.content_code  ~ ('UN|CN|EG')
-             AND   a.icp_id LIKE %(subset)s
+             AND   a.icp_id IN (
+                SELECT icp_id FROM public.icp_sample
+             )
             GROUP BY 1, 2, 3
         ) AS coup_tall
     ) AS tall_timestamp 
@@ -118,8 +106,8 @@ def getkwh(datestart, dateend, timestart, timeend, subset):
         'datestart': datestart,
         'dateend': dateend,
         'tsstart': timestart,
-        'tsend': timeend,
-        'subset': subset
+        'tsend': timeend
+        # 'subset': subset
     }
     print("Getting data with parameters:")
     print(pdict)

sql/queries.pgsql

@@ -344,3 +344,23 @@ LEFT JOIN
     ) AS coup_tall
 ) AS tall_timestamp 
 ON comb.read_time = tall_timestamp.read_time AND comb.icp_id = tall_timestamp.icp_id;
+
+-- Produces a view of all the icp's with at least 360 days of data, along with a column of days of data
+CREATE VIEW public.best_icp AS
+SELECT *
+FROM
+(
+    SELECT icp_id, COUNT(DISTINCT read_date) AS data_days 
+    FROM coup_prd.coupdatamaster
+    WHERE read_date >= to_date('01/01/2017','dd/mm/yyyy')
+        AND read_date <  to_date('01/01/2018','dd/mm/yyyy')
+        AND content_code = 'UN'
+    GROUP BY icp_id
+) AS cir 
+WHERE data_days >= 360;
+
+-- Produces sample table
+CREATE TABLE public.icp_sample AS
+SELECT * FROM public.best_icp
+ORDER BY random()
+LIMIT 1000;