Collate 5k

py/collate.py

 # Collate 12 dataframes into one (wide) combined dataframe
 import pandas as p
 import gc
+from tqdm import tqdm
 
 
 months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
 
-coldf = p.read_pickle('../data/2017-{}-all.pkl'.format(months[0]))
+coldf = p.read_pickle('../data/2017-{}-5k.pkl'.format(months[0]))
 
-for i in range(1, 12):
-    tdf = p.read_pickle('../data/2017-{}-all.pkl'.format(months[i])) 
+for i in tqdm(range(1, 12)):
+    tdf = p.read_pickle('../data/2017-{}-5k.pkl'.format(months[i])) 
     coldf = p.concat([coldf, tdf])
     del tdf
     gc.collect()
 
 print(coldf.info())
 
-coldf.to_pickle('../data/2017-all-wide.pkl')
+coldf.to_pickle('../data/2017-5k-wide.pkl')