A harmonic model for annual patterns in rolling 24-hour minimum temperature

.gitignore

@@ -1,3 +1,6 @@
+# Ignore data folder completely
+data/
+
 # ---> Python
 # Byte-compiled / optimized / DLL files
 __pycache__/

R/weathmod.R

@@ -0,0 +1,65 @@
+library(TSA)
+library(caTools)
+library(dplyr)
+library(forecast)
+library(ggplot2)
+library(reticulate)
+library(tidyr)
+theme_set(theme_bw())
+use_virtualenv("../venv/")
+
+# Load, transform data
+p <- import("pandas")
+tempdf <- p$read_pickle("../data/2016-18-weather.pkl") %>% 
+    select(-record_no, -station) %>% 
+    mutate(temp_date = as.Date(temp_date))
+str(tempdf)
+
+# Get a data frame of all the dates/times we want
+wdts <- seq.POSIXt(min(tempdf$temp_timestamp), max(tempdf$temp_timestamp), by = "30 mins")
+hhdt <- as.difftime("30", format = "%M")
+wantdf <- data.frame(temp_timestamp = wdts, temp_date = as.Date(wdts)) %>% arrange(temp_timestamp)
+str(wantdf)
+
+# Recursively impute NAs from previous non-NA value; if first value is NA replaced by median but this shouldn't happen
+# If all values are NA will hang, but this shouldn't happen either
+fulltemp <- left_join(wantdf, tempdf, by = c("temp_timestamp", "temp_date"))
+str(fulltemp)
+for (coln in names(fulltemp)[3:7]) {
+    tc <- fulltemp[[coln]]
+    if (is.na(tc[1])) {
+        print("First value of column is NA; Imputing from median")
+        tc[1] <- median(tc, na.rm = TRUE)
+    }
+    while (sum(is.na(tc)) != 0){
+        tc <- ifelse(is.na(tc), lag(tc, 1), tc)
+    }
+    fulltemp[[coln]] <- tc
+}
+sum(is.na(fulltemp$tmin_c))
+fulltemp$runmin <- runmin(fulltemp$tmin_c, 48, endrule = "min", align = "right")
+str(fulltemp)
+
+# Temperature plots
+tplot <- ggplot(fulltemp, aes(x = temp_timestamp, y = tmin_c)) + geom_line() +
+    geom_line(aes(y = runmin), color = "blue")
+
+tplot
+
+tplot + coord_cartesian(xlim = c(as.POSIXct("2017-05-01"), as.POSIXct("2017-06-01")))
+
+# Create a harmonic (sine wave) model for minimum temperature
+yharm <- harmonic(ts(1:nrow(fulltemp), frequency = floor(365.25 * 48)), 2)
+#dharm <- harmonic(ts(1:nrow(fulltemp), frequency = floor(48)), 1)
+hmod <- lm(fulltemp$runmin ~ yharm)
+summary(hmod)
+
+hmdf <- data.frame(x = fulltemp$temp_timestamp, y = fulltemp$runmin, f = fitted(hmod), r = resid(hmod))
+tmplot <- ggplot(hmdf, aes(x = x, y = y)) + geom_line(aes(y = f), color = "blue", size = 2) + geom_point() +
+    geom_point(aes(y = r), color = "darkgreen")
+
+tmplot
+
+tmplot + coord_cartesian(xlim = c(as.POSIXct("2017-05-01", tz = "UTC"), as.POSIXct("2017-06-01", tz = "UTC")))
+
+write.csv(hmdf, "../data/weatherharm.csv", row.names = FALSE)

py/downweather.py

@@ -0,0 +1,10 @@
+from util import gettemp
+import pandas as p
+
+tempdata = gettemp('2016-04-01', '2019-01-01', 2006)
+
+print(tempdata.info())
+print(tempdata.describe())
+
+print("Pickling")
+tempdata.to_pickle("../data/2016-18-weather.pkl")

py/util.py

@@ -1,6 +1,7 @@
 import psycopg2 as pg
 from configparser import ConfigParser
 import pandas.io.sql as psql
+import pandas as p
 import datetime as dt
 import numpy as np
 from pprint import pprint
@@ -120,6 +121,32 @@ def getkwh(datestart, dateend, timestart, timeend, subset):
     return(qdf)
 
 
+def gettemp(datestart, dateend, station):
+    query = """
+    SELECT record_no, station, temp_date, temp_date + temp_time AS temp_timestamp, tmax_c, tmin_c,
+        tgmin, tmean, rhmean
+    FROM weather.temperature_fact
+    WHERE station = %(station)s AND
+        temp_date >= to_date(%(datestart)s, 'yyyy-mm-dd') AND
+        temp_date <  to_date(%(dateend)s,   'yyyy-mm-dd')
+    ORDER BY temp_date, temp_time;
+    """
+    pdict = {
+        'datestart': datestart,
+        'dateend':   dateend,
+        'station':   station
+    }
+    print("Getting data with parameters:")
+    pprint(pdict)
+    qdf = getQuery(query, pdict)
+    print("converting")
+    qdf.temp_date = p.to_datetime(qdf.temp_date)
+    # qdf.temp_time = qdf.temp_time.to_timestamp()
+    print('Done')
+    return qdf
+
+
+
 
 
 if __name__ == "__main__":

sql/weather.pgsql

@@ -0,0 +1,25 @@
+-------------------------------
+-- Querying the weather data --
+-------------------------------
+
+-- Get currently operating stations
+SELECT agent, network, start_date, end_date, station_name,
+    lat_dec_deg, long_dec_deg
+FROM weather.station_dim
+WHERE end_date IS NULL AND
+    start_date <= to_date('2015-01-01', 'yyyy-mm-dd')
+ORDER BY start_date;
+
+-- 1 agent per network and vice versa?
+SELECT COUNT(DISTINCT agent) as agents, network
+FROM weather.station_dim
+GROUP BY network
+ORDER BY agents DESC;
+
+SELECT record_no, station, temp_date, temp_time, tmax_c, tmin_c,
+    tgmin, tmean, rhmean
+FROM weather.temperature_fact
+WHERE station = 2006 AND
+    temp_date >= to_date('2016-01-01', 'yyyy-mm-dd') AND
+    temp_date <  to_date('2019-01-01', 'yyyy-mm-dd')
+ORDER BY temp_date, temp_time;