ampli/py/collate.py

# Collate 12 dataframes into one (wide) combined dataframe
import pandas as p
import gc
from tqdm import tqdm


months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']

coldf = p.read_pickle('../data/2017-{}-5k.pkl'.format(months[0]))

for i in tqdm(range(1, 12)):
    tdf = p.read_pickle('../data/2017-{}-5k.pkl'.format(months[i])) 
    coldf = p.concat([coldf, tdf])
    del tdf
    gc.collect()

print(coldf.info())

coldf.to_pickle('../data/2017-5k-wide.pkl')