Make separate dcorr script

py/dcorr.py

@@ -0,0 +1,44 @@
+# Takes the correlation part out of clustering file
+from argparse import ArgumentParser
+import numpy as np
+import pandas as p
+from tqdm import tqdm
+from itertools import combinations
+from math import factorial as f
+from pprint import pprint
+
+def tqcorr(df):
+    """Calculates a (distance) correlation matrix for wide-form dataframe, with progressbar
+    """
+    cols = df.columns
+    ncols = len(cols)
+    cdf = p.DataFrame(index = cols, columns = cols, dtype = np.float16)
+    print(cdf.info())
+    for c in tqdm(cols):
+        cdf.loc[c, c] = 0
+    print(cdf.info())
+    comb = combinations(cols, 2)
+    ncomb = f(ncols) // f(2) // f(ncols - 2)
+    for c1, c2 in tqdm(comb, total = ncomb):
+        dv = 1 - df[c1].corr(df[c2])
+        cdf.loc[c1, c2] = dv
+        cdf.loc[c2, c1] = dv
+    print(cdf.info())
+    return cdf
+
+
+def createCorr(source, output):
+    """Load a pkl in wide form from source, process, run tqcorr() and save response to output
+    """
+    df = p.read_pickle(source)
+    df = df[df.columns[df.max() != df.min()]]
+    cmat = tqcorr(df)
+    cmat.to_pickle(output)
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(description='Create distance correlation matrix from pickled wideform pandas dataframe')
+    parser.add_argument("-i", "--input",  dest="input",  help = "input pickle path; default: ../data/2017-5k-wide.pkl",  metavar="[PATH]", default = "../data/2017-5k-wide.pkl")
+    parser.add_argument("-o", "--output", dest="output", help = "output pickle path; default: ../data/5kdcorrmatrix.pkl", metavar="[PATH]", default = "../data/fulldcorrmatrix.pkl")
+    args = parser.parse_args()
+    createCorr(args.input, args.output)