Save correlation matrix

py/clustering.py

@@ -8,16 +8,20 @@ import seaborn as sns
 from scipy.spatial.distance import squareform
 from scipy.cluster.hierarchy import dendrogram, linkage, cophenet, fcluster
 from tqdm import tqdm
+from itertools import combinations
 
 def tqcorr(df):
     cols = df.columns
     ncols = len(cols)
-    cdf = p.DataFrame(index = cols, columns = cols)
+    cdf = p.DataFrame(index = cols, columns = cols, dtype = np.float16)
+    print(cdf.info())
     for c in tqdm(cols):
         cdf.loc[c, c] = 0
+    print(cdf.info())
     comb = combinations(cols, 2)
     for c1, c2 in tqdm(comb):
         cdf.loc[c1, c2] = 1 - df[c1].corr(df[c2])
+    print(cdf.info())
     return cdf
 
 
@@ -47,6 +51,8 @@ cmat = tqcorr(df)
 print(cmat)
 print(cmat.info())
 
+cmat.to_pickle('../data/fulldcorrmatrix.pkl')
+
 # lmat = squareform(1 - cmat)
 
 # lobj = linkage(lmat, method = 'ward')