Algorithm for calculating correlation matrix for 5k

py/clustering.py

@@ -9,6 +9,7 @@ from scipy.spatial.distance import squareform
 from scipy.cluster.hierarchy import dendrogram, linkage, cophenet, fcluster
 from tqdm import tqdm
 from itertools import combinations
+from math import factorial as f
 
 def tqcorr(df):
     cols = df.columns
@@ -19,7 +20,8 @@ def tqcorr(df):
         cdf.loc[c, c] = 0
     print(cdf.info())
     comb = combinations(cols, 2)
-    for c1, c2 in tqdm(comb):
+    ncomb = f(ncols) // f(2) // f(ncols - 2)
+    for c1, c2 in tqdm(comb, total = ncomb):
         dv = 1 - df[c1].corr(df[c2])
         cdf.loc[c1, c2] = dv
         cdf.loc[c2, c1] = dv
@@ -29,10 +31,10 @@ def tqcorr(df):
 
 tqdm.pandas()
 
-Sourcedata =   '../data/2017-all-wide.pkl'
-lableddata =   '../data/9-clusters-all.pkl'
-aggdata =      '../data/9-clusters-all-agg.pkl'
-clustertable = '../data/9-clusters-all-table.pkl'
+Sourcedata =   '../data/2017-5k-wide.pkl'
+lableddata =   '../data/9-clusters-5k.pkl'
+aggdata =      '../data/9-clusters-5k-agg.pkl'
+clustertable = '../data/9-clusters-5k-table.pkl'
 
 numclusts = 9
 df = p.read_pickle(Sourcedata)