/readAllSymbols_daily.py

import os
import sys
import datetime as dt
import pandas
import numpy as np

def load_data_frame_from_hdf5(fullname):
    store = pandas.HDFStore(fullname, mode='r')
    output = store["data_frame"]
    store.close()
    return output

os.putenv("TZ","America/New_York")

#just one column of symbols
symbol_list_file = sys.argv[1]

#start=dt.datetime(2009,1,2,9,30)
#end=dt.datetime(2012,4,14,15,59)
symbol_list = open(symbol_list_file, 'rb')
#create a list of tickers (we'll need them anyway later)
tickers=[]
for symbol in symbol_list:
    tickers.append(symbol.rstrip('\n'))
symbol_list.close()

#read the first symbol and create starting data frames from it
#hdf5FileName = "/backtest_data/daily-1min/tbFrame1min/"  + tickers[0] + ".tb.frame.1min.hdf5"
hdf5FileName = "/backtest_data/daily-1min/tbFrameDaily/"  + tickers[0] + ".tb.frame.daily.hdf5"
print hdf5FileName
data=load_data_frame_from_hdf5(hdf5FileName)
cp=pandas.DataFrame(data.cp)
op=pandas.DataFrame(data.op)
adj_cp=pandas.DataFrame(data.adj_cp)
#load all other tickers
count=1
for symbol in tickers[1:]:
    hdf5FileName = "/backtest_data/daily-1min/tbFrameDaily/"  + symbol + ".tb.frame.daily.hdf5"
    print hdf5FileName
    data=load_data_frame_from_hdf5(hdf5FileName)
    
    cp=pandas.merge(cp, pandas.DataFrame(data.cp), left_index=True, right_index=True, how='outer')
    op=pandas.merge(op, pandas.DataFrame(data.op), left_index=True, right_index=True, how='outer')
    adj_cp=pandas.merge(adj_cp, pandas.DataFrame(data.adj_cp), left_index=True, right_index=True, how='outer')
    
    #need to have unique column names, otherwise pandas complains
    col_ind=np.arange(0,count+1)
    cp.columns=col_ind
    op.columns=col_ind
    adj_cp.columns=col_ind
    count=count+1
#rename the columns by tickers
cp.columns=tickers
op.columns=tickers
adj_cp.columns=tickers
cp.save("close_px_all_daily.bin")
op.save("open_px_all_daily.bin")
adj_cp.save("adj_close_px_all_daily.bin")