Moving from getting a longer dataset to some more decomposition stuff

R/clusterviz.R

 p <- import("pandas")
 sns <- import("seaborn")
 cbp <- as.character(p$Series(sns$color_palette("colorblind", as.integer(9))$as_hex()))
-aggdf <- p$read_pickle("../data/9-clusters.agg.pkl")
+aggdf <- p$read_pickle("../data/9-clusters-1617.agg.pkl")
 # aggdf <- as.data.frame(aggdf)
 aggdf$cluster <- factor(aggdf$cluster)
 str(aggdf)
 
 facall <- ggplot(aggdf, aes(x = read_time, y = kwh_tot_mean, color = cluster, fill = cluster)) + 
     geom_line(size = 1.5) + geom_ribbon(aes(ymin = kwh_tot_CI_low, ymax = kwh_tot_CI_high), alpha = 0.2, color = NA) +
-    labs(title = "Cluster behaviour over full year, 2017", x = "Date", y = "kwh") +
+    labs(title = "Cluster behaviour over 2016 and 2017", x = "Date", y = "kwh") +
     scale_color_manual(values = cbp) +
     scale_fill_manual(values = cbp) +
     theme(legend.position = "none") +
-    scale_x_datetime(date_breaks = "1 month", date_labels = "%-d %B")
+    scale_x_datetime(date_breaks = "1 month", date_labels = "%-d %b %y")
 
 allcon <- facall + facet_grid(cluster ~ .)
 allfre <- facall + facet_grid(cluster ~ ., scales = "free")
 octcon <- facoct + facet_grid(cluster ~ .)
 octfre <- facoct + facet_grid(cluster ~ ., scales = "free")
 
-ggsave("all-9-fix.png", allcon, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
-ggsave("all-9-fre.png", allfre, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
-ggsave("jan-9-fix.png", jancon, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
-ggsave("jan-9-fre.png", janfre, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
-ggsave("apr-9-fix.png",  apcon, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
-ggsave("apr-9-fre.png",  apfre, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
-ggsave("jul-9-fix.png", julcon, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
-ggsave("jul-9-fre.png", julfre, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
-ggsave("oct-9-fix.png", octcon, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
-ggsave("oct-9-fre.png", octfre, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
+ggsave("all-9-fix-1617.png", allcon, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
+ggsave("all-9-fre-1617.png", allfre, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
+ggsave("jan-9-fix-1617.png", jancon, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
+ggsave("jan-9-fre-1617.png", janfre, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
+ggsave("apr-9-fix-1617.png",  apcon, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
+ggsave("apr-9-fre-1617.png",  apfre, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
+ggsave("jul-9-fix-1617.png", julcon, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
+ggsave("jul-9-fre-1617.png", julfre, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
+ggsave("oct-9-fix-1617.png", octcon, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
+ggsave("oct-9-fre-1617.png", octfre, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
 
 
 # ----
     theme(legend.position = "none") + coord_cartesian(xlim = c(0, 15), expand = FALSE) +
     labs(title = "Autocorrelation plot (two weeks)", y = "Autocorrelation", x = "lag (days)")
 
-ggsave("full-autocorr.png", fcorr, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
-ggsave("week-autocorr.png", wcorr, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
+ggsave("full-autocorr-1617.png", fcorr, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
+ggsave("week-autocorr-1617.png", wcorr, path = "../img/", dpi = "retina", width = 40, height = 25, units = "cm")
 
 perd <- bind_rows(perd)
 
 plot(forecast(ctbats, h = 48 * 7 * 4))
 
 c9ts <- filter(aggdf, cluster == "9")$kwh_tot_mean
-ctsnp <- msts(c9ts, c(48, 48*7))
+ctsnp <- msts(c9ts, c(48, 48*7, 48*7*365.25))
 ctbats <- tbats(ctsnp)
-plot(forecast(ctbats, h = 48 * 7 * 4))
+plot(forecast(ctbats, h = 48 * 7))
 
 p <- periodogram(c1ts)
 dd <- data.frame(freq = p$freq, spec = p$spec) %>% mutate(per = 1/freq)
 
 ggplot(c9ts, aes(x = read_time, y = kwh_tot_mean)) + geom_line()
 
-nft <- fextract(c9ts$read_time, c9ts$kwh_tot_mean, keep = 10)
+nft <- fextract(c9ts$read_time, c9ts$kwh_tot_mean, keep = 15)
 ggplot(nft, aes(x, y)) + geom_line() + 
     geom_line(aes(x, f), color = "blue") +
     scale_x_datetime(date_breaks = "1 day", date_labels = "%a, %-d %B %Y") +
-    coord_cartesian(xlim = c(as.POSIXct("2017-07-16", tz = "UTC"), as.POSIXct("2017-07-23", tz = "UTC")), expand = TRUE)
+    coord_cartesian(xlim = c(as.POSIXct("2016-07-16", tz = "UTC"), as.POSIXct("2016-07-23", tz = "UTC")), expand = TRUE)
 
 clus <- "9"
 kp <- 50

py/clustering.py

 df = p.read_pickle('../data/2016-17-sample.pkl')
 dforig = df
 
+# print(df)
+
 print(df.info())
 print(df.icp_id.nunique())
 print(df.read_time.nunique())
 # print(df.groupby('icp_id').read_time.nunique().nunique())
 df = df.pivot(index = 'read_time', columns = 'icp_id', values = 'kwh_tot')
+print(df.info())
 df = df[df.columns[df.max() != df.min()]]
 print(df.info())
 cmat = df.corr()

py/util.py

             SELECT read_time 
             FROM GENERATE_SERIES(%(tsstart)s::timestamp, %(tsend)s::timestamp, 
                 '30 minutes'::interval) read_time
-        ) AS tsdata CROSS JOIN public.icp_sample
+        ) AS tsdata CROSS JOIN public.icp_sample_1618
     ) AS comb
     LEFT JOIN
     (
              and   a.read_date <  to_date(%(dateend)s,'yyyy-mm-dd')
              and   a.content_code  ~ ('UN|CN|EG')
              AND   a.icp_id IN (
-                SELECT icp_id FROM public.icp_sample
+                SELECT icp_id FROM public.icp_sample_1618
              )
             GROUP BY 1, 2, 3
         ) AS coup_tall

sql/queries.pgsql

 -- range of date values
 SELECT MIN(read_date) AS mindate, MAX(read_date) AS maxdate
 FROM coup_prd.coupdatamaster;
+
+-- View for best data in 18 month set
+CREATE VIEW public.best_icp_18m AS
+SELECT *
+FROM
+(
+    SELECT icp_id, COUNT(DISTINCT read_date) AS data_days 
+    FROM coup_prd.coupdatamaster
+    WHERE read_date >= to_date('01/07/2016','dd/mm/yyyy')
+        AND read_date <  to_date('01/01/2018','dd/mm/yyyy')
+        AND content_code = 'UN'
+    GROUP BY icp_id
+) AS cir 
+WHERE data_days >= 540;
+
+-- Produces sample table for 18m data
+CREATE TABLE public.icp_sample_18m AS
+SELECT * FROM public.best_icp_18m
+ORDER BY random()
+LIMIT 1000;
+
+-- Number of ICPs with data for each day
+SELECT read_date, COUNT(DISTINCT icp_id) as icps
+FROM coup_prd.coupdatamaster
+GROUP BY read_date
+ORDER BY read_date;
+
+-- View for best data in 2016-18 set
+CREATE VIEW public.best_icp_1618 AS
+SELECT *
+FROM
+(
+    SELECT icp_id, COUNT(DISTINCT read_date) AS data_days 
+    FROM coup_prd.coupdatamaster
+    WHERE read_date >= to_date('01/04/2016','dd/mm/yyyy')
+        AND read_date <  to_date('01/04/2018','dd/mm/yyyy')
+        AND content_code = 'UN'
+    GROUP BY icp_id
+) AS cir 
+WHERE data_days >= 720;
+
+-- Produces sample table for 1618 data
+CREATE TABLE public.icp_sample_1618 AS
+SELECT * FROM public.best_icp_1618
+ORDER BY random()
+LIMIT 1000;