Documentation for clustering scripts

README.md

@@ -84,3 +84,56 @@ Example:
 python downweather.py -o ../data/weathertest.pkl
 ```
 Downloads data from the default period into `../data/weathertest.pkl`.
+
+### `dcorr.py`
+
+Create a distance correlation matrix from pickled pandas dataframe. Note that this takes some time, but has a progress bar.
+
+* `-i PATH`: The path for the python "pickle" file retrieve the original data from.
+* `-o PATH`: The path for the python "pickle" file to store the result in.
+* `-p`: If the dataframe is not in wide form, pivots it so that it is first. Note: untested and unused, representing past behaviour.
+
+Example:
+
+```bash
+python dcorr.py -i ../data/test1k.pkl -o ../data/test1kdcorr.pkl
+```
+
+Calculates correlation matrix from the dataset previously downloaded to `../data/test1k.pkl` and store in `../data/test1kdcorr.pkl`.
+
+### `clustering.py`
+
+Algorithm for hierarchical clustering. **Note**: different datasets and parameters may make similar clusters in a different order.
+
+* `-i PATH`: The path for the python "pickle" file retrieve the original data from.
+* `-o PATH`: The path for the python "pickle" file to store the result in.
+* `--method METHOD`: Clustering method for hierarchical clustering; default is 'ward'. See [the scipy docs](https://docs.scipy.org/doc/scipy/reference/generated/scipy.cluster.hierarchy.linkage.html) for more information and other options.
+* `--clusters NUM`: The number of clusters; default is 9.
+* `-d`: Include dendrogram; requires `-t/--tree`.
+* `-t PATH`: Path to save dendrogram image in if `-d`
+* `-v`: Output some extra progress information as it goes; mostly useful for debugging.
+
+Example:
+
+```bash
+python clustering.py -i ../data/test1kdcorr.pkl -o ../data/test1kclustable.pkl -d -t ../img/test1kdendro.png
+```
+
+Clusters data from `../data/test1kdcorr.pkl`; saves table of clusters/icps at `../data/test1kclustable.pkl`, save dendrogram at `../img/test1kdendro.png`. 
+
+### `agg.py`
+
+Aggregates data based on clusters. **Note**: columns `CI_low` and `CI_high` do not represent formal statistical confidence intervals, merely lower and upper quartiles; for indicative purposes only. 
+
+* `-i PATH`: The path for the python "pickle" file retrieve the original data from (i.e. the data downloaded from `downkwh.py`).
+* `-c PATH`: The path for the python "pickle" file retrieve the cluster data from.
+* `-o PATH`: The path for the python "pickle" file to store the result in.
+* `-p`: If the dataframe is not in wide form, pivots it so that it is first. Note: untested and unused, representing past behaviour.
+
+Example:
+
+```bash
+python agg.py -i ../data/test1k.pkl -c ../data/test1kclustable.pkl -o ../data/test1kagg.pkl
+```
+
+Aggregates data from `../data/test1k.pkl` by cluster information in `../data/test1kclustable.pkl`, save in `../data/test1kagg.pkl`.

py/agg.py

@@ -32,9 +32,9 @@ def aggregator(widedf, clusdf):
 
 def main():
     parser = ArgumentParser(description='Aggregate dataframe by specified clusters')
-    parser.add_argument("-i", "--input",  dest="input",      help = "input pickle path; default: ../data/2017-5k-wide.pkl",  metavar="PATH", default = "../data/2017-5k-wide.pkl")
-    parser.add_argument("-c", "--clusters", dest="clusfile", help = "cluster pickle path; default: ../data/5kclustable.pkl", metavar="PATH", default = "../data/5kclustable.pkl")
-    parser.add_argument("-o", "--output", dest="output",     help = "output pickle path; default: ../data/5k-ag.pkl", metavar="PATH", default = "../data/5k-ag.pkl")
+    parser.add_argument("-i", "--input",  dest="input",      help = "input pickle path",  metavar="PATH", required = True)
+    parser.add_argument("-c", "--clusters", dest="clusfile", help = "cluster pickle path", metavar="PATH", required = True)
+    parser.add_argument("-o", "--output", dest="output",     help = "output pickle path", metavar="PATH", required = True)
     parser.add_argument("-p", "--pivot", dest = "istall",    help = "input dataframe is in tall format and must be pivoted", action ="store_true")
     args = parser.parse_args()
     wd = p.read_pickle(args.input)

py/clustering.py

@@ -55,21 +55,28 @@ def dendro(lobj, clustdf, numclusts, icps, fname):
 
 def main():
     parser = ArgumentParser(description='Cluster from pre-existing distance correlation matrix in pickled dataframe')
-    parser.add_argument("-i", "--input",  dest="input",  help = "input pickle path; default: ../data/5kdcorrmatrix.pkl",  metavar="PATH", default = "../data/5kdcorrmatrix.pkl")
-    parser.add_argument("-o", "--output", dest="output", help = "output pickle path; default: ../data/5kdclustable.pkl",  metavar="PATH", default = "../data/5kdclustable.pkl")
+    parser.add_argument("-i", "--input",  dest="input",  help = "input pickle path",  metavar="PATH", required = True)
+    parser.add_argument("-o", "--output", dest="output", help = "output pickle path",  metavar="PATH", required = True)
     parser.add_argument("--method", dest="method", help = "clustering method; default 'ward'", metavar = "METHOD", default = "ward")
     parser.add_argument("--clusters",  dest="numclusters",  help = "number of clusters; default: 9", metavar = "NUM", default = 9, type = int)
     parser.add_argument("-d", "--dendrogram", dest = "incdendro", help = "draw dendrogram", action ="store_true")
-    parser.add_argument("-t", "--tree", dest="treepath", help="Filename for dendrogram (if -d), default: ../img/59-9-dendro.png", metavar="PATH", default = "../img/5k-9-dendro.png")
+    parser.add_argument("-t", "--tree", dest="treepath", help="Filename for dendrogram (if -d)", metavar="PATH")
+    parser.add_argument("-v", "--verbose", dest = "verbose", action ="store_true")
     args = parser.parse_args()
 
-    print("Clustering")
+    if args.incdendro and args.treepath is None:
+        parser.error("-d/--dendrogram requires -t/--tree PATH")
+
+    if args.verbose:
+        print("Clustering")
+
     sourcep = p.read_pickle(args.input)
     l, c = cluster(sourcep, args.method, args.numclusters)
     c.to_pickle(args.output)
 
-    print("Drawing dendrogram")
     if args.incdendro:
+        if args.verbose:
+            print("Drawing dendrogram")
         icps = sourcep.index.values
         dendro(l, c, args.numclusters, icps, args.treepath)
 

py/dcorr.py

@@ -36,8 +36,8 @@ def createCorr(source, output, piv):
 
 if __name__ == "__main__":
     parser = ArgumentParser(description='Create distance correlation matrix from pickled wideform pandas dataframe')
-    parser.add_argument("-i", "--input",  dest="input",   help = "input pickle path; default: ../data/2017-5k-wide.pkl",  metavar="PATH", default = "../data/2017-5k-wide.pkl")
-    parser.add_argument("-o", "--output", dest="output",  help = "output pickle path; default: ../data/5kdcorrmatrix.pkl", metavar="PATH", default = "../data/5kdcorrmatrix.pkl")
+    parser.add_argument("-i", "--input",  dest="input",   help = "input pickle path",  metavar="PATH", required = True)
+    parser.add_argument("-o", "--output", dest="output",  help = "output pickle path", metavar="PATH", required = True)
     parser.add_argument("-p", "--pivot", dest = "istall", help = "input dataframe is in tall format and must be pivoted", action ="store_true")
     args = parser.parse_args()
     createCorr(args.input, args.output, args.istall)