/python/algo_local/read_sentiment.py

import datetime as dt
import pandas
import numpy as np
import os

#set time zone, otherwise dates are screwed
os.putenv("TZ","America/New_York")

path="/backtest_data/sentiment_data/recorded_future/"

rf=pandas.read_csv(path+'RF-R3000-HalfHour-History-Attention.csv', sep=',', index_col=[0,2], parse_dates=True)
'''
the data is MultyIndex pandas data frame. First index is ticker,
second index is date
'''

#don't need RF_ID column
rf.pop('RF_ID')

#add weighted sentiment
rf['w30']=(rf.Positive-rf.Negative)*rf['Attention.30min']
rf['w24h']=(rf.Positive-rf.Negative)*rf['Attention.24hr']

path="/backtest_data/1second/teams_data/team3/data/"
rf.save(path+"recorded_future.bin")

#sentiment.Count.ix['AAPL'].ix[dt.datetime(2011,1,2):dt.datetime(2011,1,3)].plot()