/pandas/io/tests/test_pytables.py

https://github.com/thouis/pandas · Python · 1333 lines · 973 code · 263 blank · 97 comment · 58 complexity · b519a9b65dbf472aeffe6615ed78b1e0 MD5 · raw file

  1. import nose
  2. import unittest
  3. import os
  4. import sys
  5. import warnings
  6. from datetime import datetime
  7. import numpy as np
  8. from pandas import (Series, DataFrame, Panel, MultiIndex, bdate_range,
  9. date_range, Index)
  10. from pandas.io.pytables import HDFStore, get_store, Term, IncompatibilityWarning
  11. import pandas.util.testing as tm
  12. from pandas.tests.test_series import assert_series_equal
  13. from pandas.tests.test_frame import assert_frame_equal
  14. from pandas import concat, Timestamp
  15. try:
  16. import tables
  17. except ImportError:
  18. raise nose.SkipTest('no pytables')
  19. from distutils.version import LooseVersion
  20. _default_compressor = LooseVersion(tables.__version__) >= '2.2' \
  21. and 'blosc' or 'zlib'
  22. _multiprocess_can_split_ = False
  23. class TestHDFStore(unittest.TestCase):
  24. path = '__test__.h5'
  25. scratchpath = '__scratch__.h5'
  26. def setUp(self):
  27. self.store = HDFStore(self.path)
  28. def tearDown(self):
  29. self.store.close()
  30. os.remove(self.path)
  31. def test_factory_fun(self):
  32. try:
  33. with get_store(self.scratchpath) as tbl:
  34. raise ValueError('blah')
  35. except ValueError:
  36. pass
  37. with get_store(self.scratchpath) as tbl:
  38. tbl['a'] = tm.makeDataFrame()
  39. with get_store(self.scratchpath) as tbl:
  40. self.assertEquals(len(tbl), 1)
  41. self.assertEquals(type(tbl['a']), DataFrame)
  42. os.remove(self.scratchpath)
  43. def test_keys(self):
  44. self.store['a'] = tm.makeTimeSeries()
  45. self.store['b'] = tm.makeStringSeries()
  46. self.store['c'] = tm.makeDataFrame()
  47. self.store['d'] = tm.makePanel()
  48. self.store['foo/bar'] = tm.makePanel()
  49. self.assertEquals(len(self.store), 5)
  50. self.assert_(set(self.store.keys()) == set(['/a', '/b', '/c', '/d', '/foo/bar']))
  51. def test_repr(self):
  52. repr(self.store)
  53. self.store['a'] = tm.makeTimeSeries()
  54. self.store['b'] = tm.makeStringSeries()
  55. self.store['c'] = tm.makeDataFrame()
  56. self.store['d'] = tm.makePanel()
  57. self.store['foo/bar'] = tm.makePanel()
  58. self.store.append('e', tm.makePanel())
  59. repr(self.store)
  60. str(self.store)
  61. def test_contains(self):
  62. self.store['a'] = tm.makeTimeSeries()
  63. self.store['b'] = tm.makeDataFrame()
  64. self.store['foo/bar'] = tm.makeDataFrame()
  65. self.assert_('a' in self.store)
  66. self.assert_('b' in self.store)
  67. self.assert_('c' not in self.store)
  68. self.assert_('foo/bar' in self.store)
  69. self.assert_('/foo/bar' in self.store)
  70. self.assert_('/foo/b' not in self.store)
  71. self.assert_('bar' not in self.store)
  72. def test_versioning(self):
  73. self.store['a'] = tm.makeTimeSeries()
  74. self.store['b'] = tm.makeDataFrame()
  75. df = tm.makeTimeDataFrame()
  76. self.store.remove('df1')
  77. self.store.append('df1', df[:10])
  78. self.store.append('df1', df[10:])
  79. self.assert_(self.store.root.a._v_attrs.pandas_version == '0.10')
  80. self.assert_(self.store.root.b._v_attrs.pandas_version == '0.10')
  81. self.assert_(self.store.root.df1._v_attrs.pandas_version == '0.10')
  82. # write a file and wipe its versioning
  83. self.store.remove('df2')
  84. self.store.append('df2', df)
  85. self.store.get_node('df2')._v_attrs.pandas_version = None
  86. self.store.select('df2')
  87. self.store.select('df2', [ Term('index','>',df.index[2]) ])
  88. def test_meta(self):
  89. raise nose.SkipTest('no meta')
  90. meta = { 'foo' : [ 'I love pandas ' ] }
  91. s = tm.makeTimeSeries()
  92. s.meta = meta
  93. self.store['a'] = s
  94. self.assert_(self.store['a'].meta == meta)
  95. df = tm.makeDataFrame()
  96. df.meta = meta
  97. self.store['b'] = df
  98. self.assert_(self.store['b'].meta == meta)
  99. # this should work, but because slicing doesn't propgate meta it doesn
  100. self.store.remove('df1')
  101. self.store.append('df1', df[:10])
  102. self.store.append('df1', df[10:])
  103. results = self.store['df1']
  104. #self.assert_(getattr(results,'meta',None) == meta)
  105. # no meta
  106. df = tm.makeDataFrame()
  107. self.store['b'] = df
  108. self.assert_(hasattr(self.store['b'],'meta') == False)
  109. def test_reopen_handle(self):
  110. self.store['a'] = tm.makeTimeSeries()
  111. self.store.open('w', warn=False)
  112. self.assert_(self.store.handle.isopen)
  113. self.assertEquals(len(self.store), 0)
  114. def test_flush(self):
  115. self.store['a'] = tm.makeTimeSeries()
  116. self.store.flush()
  117. def test_get(self):
  118. self.store['a'] = tm.makeTimeSeries()
  119. left = self.store.get('a')
  120. right = self.store['a']
  121. tm.assert_series_equal(left, right)
  122. left = self.store.get('/a')
  123. right = self.store['/a']
  124. tm.assert_series_equal(left, right)
  125. self.assertRaises(KeyError, self.store.get, 'b')
  126. def test_put(self):
  127. ts = tm.makeTimeSeries()
  128. df = tm.makeTimeDataFrame()
  129. self.store['a'] = ts
  130. self.store['b'] = df[:10]
  131. self.store['foo/bar/bah'] = df[:10]
  132. self.store['foo'] = df[:10]
  133. self.store['/foo'] = df[:10]
  134. self.store.put('c', df[:10], table=True)
  135. # not OK, not a table
  136. self.assertRaises(ValueError, self.store.put, 'b', df[10:], append=True)
  137. # node does not currently exist, test _is_table_type returns False in
  138. # this case
  139. self.assertRaises(ValueError, self.store.put, 'f', df[10:], append=True)
  140. # OK
  141. self.store.put('c', df[10:], append=True)
  142. # overwrite table
  143. self.store.put('c', df[:10], table=True, append=False)
  144. tm.assert_frame_equal(df[:10], self.store['c'])
  145. def test_put_string_index(self):
  146. index = Index([ "I am a very long string index: %s" % i for i in range(20) ])
  147. s = Series(np.arange(20), index = index)
  148. df = DataFrame({ 'A' : s, 'B' : s })
  149. self.store['a'] = s
  150. tm.assert_series_equal(self.store['a'], s)
  151. self.store['b'] = df
  152. tm.assert_frame_equal(self.store['b'], df)
  153. # mixed length
  154. index = Index(['abcdefghijklmnopqrstuvwxyz1234567890'] + [ "I am a very long string index: %s" % i for i in range(20) ])
  155. s = Series(np.arange(21), index = index)
  156. df = DataFrame({ 'A' : s, 'B' : s })
  157. self.store['a'] = s
  158. tm.assert_series_equal(self.store['a'], s)
  159. self.store['b'] = df
  160. tm.assert_frame_equal(self.store['b'], df)
  161. def test_put_compression(self):
  162. df = tm.makeTimeDataFrame()
  163. self.store.put('c', df, table=True, compression='zlib')
  164. tm.assert_frame_equal(self.store['c'], df)
  165. # can't compress if table=False
  166. self.assertRaises(ValueError, self.store.put, 'b', df,
  167. table=False, compression='zlib')
  168. def test_put_compression_blosc(self):
  169. tm.skip_if_no_package('tables', '2.2', app='blosc support')
  170. df = tm.makeTimeDataFrame()
  171. # can't compress if table=False
  172. self.assertRaises(ValueError, self.store.put, 'b', df,
  173. table=False, compression='blosc')
  174. self.store.put('c', df, table=True, compression='blosc')
  175. tm.assert_frame_equal(self.store['c'], df)
  176. def test_put_integer(self):
  177. # non-date, non-string index
  178. df = DataFrame(np.random.randn(50, 100))
  179. self._check_roundtrip(df, tm.assert_frame_equal)
  180. def test_append(self):
  181. df = tm.makeTimeDataFrame()
  182. self.store.remove('df1')
  183. self.store.append('df1', df[:10])
  184. self.store.append('df1', df[10:])
  185. tm.assert_frame_equal(self.store['df1'], df)
  186. self.store.remove('df2')
  187. self.store.put('df2', df[:10], table=True)
  188. self.store.append('df2', df[10:])
  189. tm.assert_frame_equal(self.store['df2'], df)
  190. self.store.remove('df3')
  191. self.store.append('/df3', df[:10])
  192. self.store.append('/df3', df[10:])
  193. tm.assert_frame_equal(self.store['df3'], df)
  194. # this is allowed by almost always don't want to do it
  195. warnings.filterwarnings('ignore', category=tables.NaturalNameWarning)
  196. self.store.remove('/df3 foo')
  197. self.store.append('/df3 foo', df[:10])
  198. self.store.append('/df3 foo', df[10:])
  199. tm.assert_frame_equal(self.store['df3 foo'], df)
  200. warnings.filterwarnings('always', category=tables.NaturalNameWarning)
  201. # panel
  202. wp = tm.makePanel()
  203. self.store.remove('wp1')
  204. self.store.append('wp1', wp.ix[:,:10,:])
  205. self.store.append('wp1', wp.ix[:,10:,:])
  206. tm.assert_panel_equal(self.store['wp1'], wp)
  207. # ndim
  208. p4d = tm.makePanel4D()
  209. self.store.remove('p4d')
  210. self.store.append('p4d', p4d.ix[:,:,:10,:])
  211. self.store.append('p4d', p4d.ix[:,:,10:,:])
  212. tm.assert_panel4d_equal(self.store['p4d'], p4d)
  213. # test using axis labels
  214. self.store.remove('p4d')
  215. self.store.append('p4d', p4d.ix[:,:,:10,:], axes=['items','major_axis','minor_axis'])
  216. self.store.append('p4d', p4d.ix[:,:,10:,:], axes=['items','major_axis','minor_axis'])
  217. tm.assert_panel4d_equal(self.store['p4d'], p4d)
  218. # test using differnt number of items on each axis
  219. p4d2 = p4d.copy()
  220. p4d2['l4'] = p4d['l1']
  221. p4d2['l5'] = p4d['l1']
  222. self.store.remove('p4d2')
  223. self.store.append('p4d2', p4d2, axes=['items','major_axis','minor_axis'])
  224. tm.assert_panel4d_equal(self.store['p4d2'], p4d2)
  225. # test using differt order of items on the non-index axes
  226. self.store.remove('wp1')
  227. wp_append1 = wp.ix[:,:10,:]
  228. self.store.append('wp1', wp_append1)
  229. wp_append2 = wp.ix[:,10:,:].reindex(items = wp.items[::-1])
  230. self.store.append('wp1', wp_append2)
  231. tm.assert_panel_equal(self.store['wp1'], wp)
  232. def test_append_frame_column_oriented(self):
  233. # column oriented
  234. df = tm.makeTimeDataFrame()
  235. self.store.remove('df1')
  236. self.store.append('df1', df.ix[:,:2], axes = ['columns'])
  237. self.store.append('df1', df.ix[:,2:])
  238. tm.assert_frame_equal(self.store['df1'], df)
  239. result = self.store.select('df1', 'columns=A')
  240. expected = df.reindex(columns=['A'])
  241. tm.assert_frame_equal(expected, result)
  242. # this isn't supported
  243. self.assertRaises(Exception, self.store.select, 'df1', ('columns=A', Term('index','>',df.index[4])))
  244. # selection on the non-indexable
  245. result = self.store.select('df1', ('columns=A', Term('index','=',df.index[0:4])))
  246. expected = df.reindex(columns=['A'],index=df.index[0:4])
  247. tm.assert_frame_equal(expected, result)
  248. def test_ndim_indexables(self):
  249. """ test using ndim tables in new ways"""
  250. p4d = tm.makePanel4D()
  251. def check_indexers(key, indexers):
  252. for i,idx in enumerate(indexers):
  253. self.assert_(getattr(getattr(self.store.root,key).table.description,idx)._v_pos == i)
  254. # append then change (will take existing schema)
  255. indexers = ['items','major_axis','minor_axis']
  256. self.store.remove('p4d')
  257. self.store.append('p4d', p4d.ix[:,:,:10,:], axes=indexers)
  258. self.store.append('p4d', p4d.ix[:,:,10:,:])
  259. tm.assert_panel4d_equal(self.store.select('p4d'),p4d)
  260. check_indexers('p4d',indexers)
  261. # same as above, but try to append with differnt axes
  262. self.store.remove('p4d')
  263. self.store.append('p4d', p4d.ix[:,:,:10,:], axes=indexers)
  264. self.store.append('p4d', p4d.ix[:,:,10:,:], axes=['labels','items','major_axis'])
  265. tm.assert_panel4d_equal(self.store.select('p4d'),p4d)
  266. check_indexers('p4d',indexers)
  267. # pass incorrect number of axes
  268. self.store.remove('p4d')
  269. self.assertRaises(Exception, self.store.append, 'p4d', p4d.ix[:,:,:10,:], axes=['major_axis','minor_axis'])
  270. # different than default indexables #1
  271. indexers = ['labels','major_axis','minor_axis']
  272. self.store.remove('p4d')
  273. self.store.append('p4d', p4d.ix[:,:,:10,:], axes=indexers)
  274. self.store.append('p4d', p4d.ix[:,:,10:,:])
  275. tm.assert_panel4d_equal(self.store['p4d'], p4d)
  276. check_indexers('p4d',indexers)
  277. # different than default indexables #2
  278. indexers = ['major_axis','labels','minor_axis']
  279. self.store.remove('p4d')
  280. self.store.append('p4d', p4d.ix[:,:,:10,:], axes=indexers)
  281. self.store.append('p4d', p4d.ix[:,:,10:,:])
  282. tm.assert_panel4d_equal(self.store['p4d'], p4d)
  283. check_indexers('p4d',indexers)
  284. # partial selection
  285. result = self.store.select('p4d',['labels=l1'])
  286. expected = p4d.reindex(labels = ['l1'])
  287. tm.assert_panel4d_equal(result, expected)
  288. # partial selection2
  289. result = self.store.select('p4d',[Term('labels=l1'), Term('items=ItemA'), Term('minor_axis=B')])
  290. expected = p4d.reindex(labels = ['l1'], items = ['ItemA'], minor_axis = ['B'])
  291. tm.assert_panel4d_equal(result, expected)
  292. # non-existant partial selection
  293. result = self.store.select('p4d',[Term('labels=l1'), Term('items=Item1'), Term('minor_axis=B')])
  294. expected = p4d.reindex(labels = ['l1'], items = [], minor_axis = ['B'])
  295. tm.assert_panel4d_equal(result, expected)
  296. def test_append_with_strings(self):
  297. wp = tm.makePanel()
  298. wp2 = wp.rename_axis(dict([ (x,"%s_extra" % x) for x in wp.minor_axis ]), axis = 2)
  299. self.store.append('s1', wp, min_itemsize = 20)
  300. self.store.append('s1', wp2)
  301. expected = concat([ wp, wp2], axis = 2)
  302. expected = expected.reindex(minor_axis = sorted(expected.minor_axis))
  303. tm.assert_panel_equal(self.store['s1'], expected)
  304. # test dict format
  305. self.store.append('s2', wp, min_itemsize = { 'minor_axis' : 20 })
  306. self.store.append('s2', wp2)
  307. expected = concat([ wp, wp2], axis = 2)
  308. expected = expected.reindex(minor_axis = sorted(expected.minor_axis))
  309. tm.assert_panel_equal(self.store['s2'], expected)
  310. # apply the wrong field (similar to #1)
  311. self.store.append('s3', wp, min_itemsize = { 'major_axis' : 20 })
  312. self.assertRaises(Exception, self.store.append, 's3')
  313. # test truncation of bigger strings
  314. self.store.append('s4', wp)
  315. self.assertRaises(Exception, self.store.append, 's4', wp2)
  316. # avoid truncation on elements
  317. df = DataFrame([[123,'asdqwerty'], [345,'dggnhebbsdfbdfb']])
  318. self.store.append('df_big',df, min_itemsize = { 'values' : 1024 })
  319. tm.assert_frame_equal(self.store.select('df_big'), df)
  320. # appending smaller string ok
  321. df2 = DataFrame([[124,'asdqy'], [346,'dggnhefbdfb']])
  322. self.store.append('df_big',df2)
  323. expected = concat([ df, df2 ])
  324. tm.assert_frame_equal(self.store.select('df_big'), expected)
  325. # avoid truncation on elements
  326. df = DataFrame([[123,'asdqwerty'], [345,'dggnhebbsdfbdfb']])
  327. self.store.append('df_big2',df, min_itemsize = { 'values' : 10 })
  328. tm.assert_frame_equal(self.store.select('df_big2'), df)
  329. # bigger string on next append
  330. self.store.append('df_new',df, min_itemsize = { 'values' : 16 })
  331. df_new = DataFrame([[124,'abcdefqhij'], [346, 'abcdefghijklmnopqrtsuvwxyz']])
  332. self.assertRaises(Exception, self.store.append, 'df_new',df_new)
  333. def test_create_table_index(self):
  334. wp = tm.makePanel()
  335. self.store.append('p5', wp)
  336. self.store.create_table_index('p5')
  337. assert(self.store.handle.root.p5.table.cols.major_axis.is_indexed == True)
  338. assert(self.store.handle.root.p5.table.cols.minor_axis.is_indexed == False)
  339. # default optlevels
  340. assert(self.store.handle.root.p5.table.cols.major_axis.index.optlevel == 6)
  341. assert(self.store.handle.root.p5.table.cols.major_axis.index.kind == 'medium')
  342. # let's change the indexing scheme
  343. self.store.create_table_index('p5')
  344. assert(self.store.handle.root.p5.table.cols.major_axis.index.optlevel == 6)
  345. assert(self.store.handle.root.p5.table.cols.major_axis.index.kind == 'medium')
  346. self.store.create_table_index('p5', optlevel=9)
  347. assert(self.store.handle.root.p5.table.cols.major_axis.index.optlevel == 9)
  348. assert(self.store.handle.root.p5.table.cols.major_axis.index.kind == 'medium')
  349. self.store.create_table_index('p5', kind='full')
  350. assert(self.store.handle.root.p5.table.cols.major_axis.index.optlevel == 9)
  351. assert(self.store.handle.root.p5.table.cols.major_axis.index.kind == 'full')
  352. self.store.create_table_index('p5', optlevel=1, kind='light')
  353. assert(self.store.handle.root.p5.table.cols.major_axis.index.optlevel == 1)
  354. assert(self.store.handle.root.p5.table.cols.major_axis.index.kind == 'light')
  355. df = tm.makeTimeDataFrame()
  356. self.store.append('f', df[:10])
  357. self.store.append('f', df[10:])
  358. self.store.create_table_index('f')
  359. # try to index a non-table
  360. self.store.put('f2', df)
  361. self.assertRaises(Exception, self.store.create_table_index, 'f2')
  362. # try to change the version supports flag
  363. from pandas.io import pytables
  364. pytables._table_supports_index = False
  365. self.assertRaises(Exception, self.store.create_table_index, 'f')
  366. # test out some versions
  367. original = tables.__version__
  368. for v in ['2.2','2.2b']:
  369. pytables._table_mod = None
  370. pytables._table_supports_index = False
  371. tables.__version__ = v
  372. self.assertRaises(Exception, self.store.create_table_index, 'f')
  373. for v in ['2.3.1','2.3.1b','2.4dev','2.4',original]:
  374. pytables._table_mod = None
  375. pytables._table_supports_index = False
  376. tables.__version__ = v
  377. self.store.create_table_index('f')
  378. pytables._table_mod = None
  379. pytables._table_supports_index = False
  380. tables.__version__ = original
  381. def test_big_table(self):
  382. raise nose.SkipTest('no big table')
  383. # create and write a big table
  384. wp = Panel(np.random.randn(20, 1000, 1000), items= [ 'Item%s' % i for i in xrange(20) ],
  385. major_axis=date_range('1/1/2000', periods=1000), minor_axis = [ 'E%s' % i for i in xrange(1000) ])
  386. wp.ix[:,100:200,300:400] = np.nan
  387. try:
  388. store = HDFStore(self.scratchpath)
  389. store._debug_memory = True
  390. store.append('wp',wp)
  391. recons = store.select('wp')
  392. finally:
  393. store.close()
  394. os.remove(self.scratchpath)
  395. def test_append_diff_item_order(self):
  396. raise nose.SkipTest('append diff item order')
  397. wp = tm.makePanel()
  398. wp1 = wp.ix[:, :10, :]
  399. wp2 = wp.ix[['ItemC', 'ItemB', 'ItemA'], 10:, :]
  400. self.store.put('panel', wp1, table=True)
  401. self.assertRaises(Exception, self.store.put, 'panel', wp2,
  402. append=True)
  403. def test_table_index_incompatible_dtypes(self):
  404. df1 = DataFrame({'a': [1, 2, 3]})
  405. df2 = DataFrame({'a': [4, 5, 6]},
  406. index=date_range('1/1/2000', periods=3))
  407. self.store.put('frame', df1, table=True)
  408. self.assertRaises(Exception, self.store.put, 'frame', df2,
  409. table=True, append=True)
  410. def test_table_values_dtypes_roundtrip(self):
  411. df1 = DataFrame({'a': [1, 2, 3]}, dtype = 'f8')
  412. self.store.append('df1', df1)
  413. assert df1.dtypes == self.store['df1'].dtypes
  414. df2 = DataFrame({'a': [1, 2, 3]}, dtype = 'i8')
  415. self.store.append('df2', df2)
  416. assert df2.dtypes == self.store['df2'].dtypes
  417. # incompatible dtype
  418. self.assertRaises(Exception, self.store.append, 'df2', df1)
  419. def test_table_mixed_dtypes(self):
  420. # frame
  421. def _make_one_df():
  422. df = tm.makeDataFrame()
  423. df['obj1'] = 'foo'
  424. df['obj2'] = 'bar'
  425. df['bool1'] = df['A'] > 0
  426. df['bool2'] = df['B'] > 0
  427. df['bool3'] = True
  428. df['int1'] = 1
  429. df['int2'] = 2
  430. return df.consolidate()
  431. df1 = _make_one_df()
  432. self.store.append('df1_mixed', df1)
  433. tm.assert_frame_equal(self.store.select('df1_mixed'), df1)
  434. # panel
  435. def _make_one_panel():
  436. wp = tm.makePanel()
  437. wp['obj1'] = 'foo'
  438. wp['obj2'] = 'bar'
  439. wp['bool1'] = wp['ItemA'] > 0
  440. wp['bool2'] = wp['ItemB'] > 0
  441. wp['int1'] = 1
  442. wp['int2'] = 2
  443. return wp.consolidate()
  444. p1 = _make_one_panel()
  445. self.store.append('p1_mixed', p1)
  446. tm.assert_panel_equal(self.store.select('p1_mixed'), p1)
  447. # ndim
  448. def _make_one_p4d():
  449. wp = tm.makePanel4D()
  450. wp['obj1'] = 'foo'
  451. wp['obj2'] = 'bar'
  452. wp['bool1'] = wp['l1'] > 0
  453. wp['bool2'] = wp['l2'] > 0
  454. wp['int1'] = 1
  455. wp['int2'] = 2
  456. return wp.consolidate()
  457. p4d = _make_one_p4d()
  458. self.store.append('p4d_mixed', p4d)
  459. tm.assert_panel4d_equal(self.store.select('p4d_mixed'), p4d)
  460. def test_remove(self):
  461. ts = tm.makeTimeSeries()
  462. df = tm.makeDataFrame()
  463. self.store['a'] = ts
  464. self.store['b'] = df
  465. self.store.remove('a')
  466. self.assertEquals(len(self.store), 1)
  467. tm.assert_frame_equal(df, self.store['b'])
  468. self.store.remove('b')
  469. self.assertEquals(len(self.store), 0)
  470. # pathing
  471. self.store['a'] = ts
  472. self.store['b/foo'] = df
  473. self.store.remove('foo')
  474. self.store.remove('b/foo')
  475. self.assertEquals(len(self.store), 1)
  476. self.store['a'] = ts
  477. self.store['b/foo'] = df
  478. self.store.remove('b')
  479. self.assertEquals(len(self.store), 1)
  480. # __delitem__
  481. self.store['a'] = ts
  482. self.store['b'] = df
  483. del self.store['a']
  484. del self.store['b']
  485. self.assertEquals(len(self.store), 0)
  486. def test_remove_where(self):
  487. # non-existance
  488. crit1 = Term('index','>','foo')
  489. self.store.remove('a', where=[crit1])
  490. # try to remove non-table (with crit)
  491. # non-table ok (where = None)
  492. wp = tm.makePanel()
  493. self.store.put('wp', wp, table=True)
  494. self.store.remove('wp', [('minor_axis', ['A', 'D'])])
  495. rs = self.store.select('wp')
  496. expected = wp.reindex(minor_axis = ['B','C'])
  497. tm.assert_panel_equal(rs,expected)
  498. # empty where
  499. self.store.remove('wp')
  500. self.store.put('wp', wp, table=True)
  501. # deleted number (entire table)
  502. n = self.store.remove('wp', [])
  503. assert(n == 120)
  504. # non - empty where
  505. self.store.remove('wp')
  506. self.store.put('wp', wp, table=True)
  507. self.assertRaises(Exception, self.store.remove,
  508. 'wp', ['foo'])
  509. # selectin non-table with a where
  510. #self.store.put('wp2', wp, table=False)
  511. #self.assertRaises(Exception, self.store.remove,
  512. # 'wp2', [('column', ['A', 'D'])])
  513. def test_remove_crit(self):
  514. wp = tm.makePanel()
  515. # group row removal
  516. date4 = wp.major_axis.take([ 0,1,2,4,5,6,8,9,10 ])
  517. crit4 = Term('major_axis',date4)
  518. self.store.put('wp3', wp, table=True)
  519. n = self.store.remove('wp3', where=[crit4])
  520. assert(n == 36)
  521. result = self.store.select('wp3')
  522. expected = wp.reindex(major_axis = wp.major_axis-date4)
  523. tm.assert_panel_equal(result, expected)
  524. # upper half
  525. self.store.put('wp', wp, table=True)
  526. date = wp.major_axis[len(wp.major_axis) // 2]
  527. crit1 = Term('major_axis','>',date)
  528. crit2 = Term('minor_axis',['A', 'D'])
  529. n = self.store.remove('wp', where=[crit1])
  530. assert(n == 56)
  531. n = self.store.remove('wp', where=[crit2])
  532. assert(n == 32)
  533. result = self.store['wp']
  534. expected = wp.truncate(after=date).reindex(minor=['B', 'C'])
  535. tm.assert_panel_equal(result, expected)
  536. # individual row elements
  537. self.store.put('wp2', wp, table=True)
  538. date1 = wp.major_axis[1:3]
  539. crit1 = Term('major_axis',date1)
  540. self.store.remove('wp2', where=[crit1])
  541. result = self.store.select('wp2')
  542. expected = wp.reindex(major_axis=wp.major_axis-date1)
  543. tm.assert_panel_equal(result, expected)
  544. date2 = wp.major_axis[5]
  545. crit2 = Term('major_axis',date2)
  546. self.store.remove('wp2', where=[crit2])
  547. result = self.store['wp2']
  548. expected = wp.reindex(major_axis=wp.major_axis-date1-Index([date2]))
  549. tm.assert_panel_equal(result, expected)
  550. date3 = [wp.major_axis[7],wp.major_axis[9]]
  551. crit3 = Term('major_axis',date3)
  552. self.store.remove('wp2', where=[crit3])
  553. result = self.store['wp2']
  554. expected = wp.reindex(major_axis=wp.major_axis-date1-Index([date2])-Index(date3))
  555. tm.assert_panel_equal(result, expected)
  556. # corners
  557. self.store.put('wp4', wp, table=True)
  558. n = self.store.remove('wp4', where=[Term('major_axis','>',wp.major_axis[-1])])
  559. result = self.store.select('wp4')
  560. tm.assert_panel_equal(result, wp)
  561. def test_terms(self):
  562. wp = tm.makePanel()
  563. p4d = tm.makePanel4D()
  564. self.store.put('wp', wp, table=True)
  565. self.store.put('p4d', p4d, table=True)
  566. # some invalid terms
  567. terms = [
  568. [ 'minor', ['A','B'] ],
  569. [ 'index', ['20121114'] ],
  570. [ 'index', ['20121114', '20121114'] ],
  571. ]
  572. for t in terms:
  573. self.assertRaises(Exception, self.store.select, 'wp', t)
  574. self.assertRaises(Exception, Term.__init__)
  575. self.assertRaises(Exception, Term.__init__, 'blah')
  576. self.assertRaises(Exception, Term.__init__, 'index')
  577. self.assertRaises(Exception, Term.__init__, 'index', '==')
  578. self.assertRaises(Exception, Term.__init__, 'index', '>', 5)
  579. # panel
  580. result = self.store.select('wp',[ Term('major_axis<20000108'), Term('minor_axis', '=', ['A','B']) ])
  581. expected = wp.truncate(after='20000108').reindex(minor=['A', 'B'])
  582. tm.assert_panel_equal(result, expected)
  583. # p4d
  584. result = self.store.select('p4d',[ Term('major_axis<20000108'), Term('minor_axis', '=', ['A','B']), Term('items', '=', ['ItemA','ItemB']) ])
  585. expected = p4d.truncate(after='20000108').reindex(minor=['A', 'B'],items=['ItemA','ItemB'])
  586. tm.assert_panel4d_equal(result, expected)
  587. # valid terms
  588. terms = [
  589. dict(field = 'major_axis', op = '>', value = '20121114'),
  590. ('major_axis', '20121114'),
  591. ('major_axis', '>', '20121114'),
  592. (('major_axis', ['20121114','20121114']),),
  593. ('major_axis', datetime(2012,11,14)),
  594. 'major_axis>20121114',
  595. 'major_axis>20121114',
  596. 'major_axis>20121114',
  597. (('minor_axis', ['A','B']),),
  598. (('minor_axis', ['A','B']),),
  599. ((('minor_axis', ['A','B']),),),
  600. (('items', ['ItemA','ItemB']),),
  601. ('items=ItemA'),
  602. ]
  603. for t in terms:
  604. self.store.select('wp', t)
  605. self.store.select('p4d', t)
  606. # valid for p4d only
  607. terms = [
  608. (('labels', '=', ['l1','l2']),),
  609. Term('labels', '=', ['l1','l2']),
  610. ]
  611. for t in terms:
  612. self.store.select('p4d', t)
  613. def test_series(self):
  614. s = tm.makeStringSeries()
  615. self._check_roundtrip(s, tm.assert_series_equal)
  616. ts = tm.makeTimeSeries()
  617. self._check_roundtrip(ts, tm.assert_series_equal)
  618. ts2 = Series(ts.index, Index(ts.index, dtype=object))
  619. self._check_roundtrip(ts2, tm.assert_series_equal)
  620. ts3 = Series(ts.values, Index(np.asarray(ts.index, dtype=object),
  621. dtype=object))
  622. self._check_roundtrip(ts3, tm.assert_series_equal)
  623. def test_sparse_series(self):
  624. s = tm.makeStringSeries()
  625. s[3:5] = np.nan
  626. ss = s.to_sparse()
  627. self._check_roundtrip(ss, tm.assert_series_equal,
  628. check_series_type=True)
  629. ss2 = s.to_sparse(kind='integer')
  630. self._check_roundtrip(ss2, tm.assert_series_equal,
  631. check_series_type=True)
  632. ss3 = s.to_sparse(fill_value=0)
  633. self._check_roundtrip(ss3, tm.assert_series_equal,
  634. check_series_type=True)
  635. def test_sparse_frame(self):
  636. s = tm.makeDataFrame()
  637. s.ix[3:5, 1:3] = np.nan
  638. s.ix[8:10, -2] = np.nan
  639. ss = s.to_sparse()
  640. self._check_double_roundtrip(ss, tm.assert_frame_equal,
  641. check_frame_type=True)
  642. ss2 = s.to_sparse(kind='integer')
  643. self._check_double_roundtrip(ss2, tm.assert_frame_equal,
  644. check_frame_type=True)
  645. ss3 = s.to_sparse(fill_value=0)
  646. self._check_double_roundtrip(ss3, tm.assert_frame_equal,
  647. check_frame_type=True)
  648. def test_sparse_panel(self):
  649. items = ['x', 'y', 'z']
  650. p = Panel(dict((i, tm.makeDataFrame().ix[:2, :2]) for i in items))
  651. sp = p.to_sparse()
  652. self._check_double_roundtrip(sp, tm.assert_panel_equal,
  653. check_panel_type=True)
  654. sp2 = p.to_sparse(kind='integer')
  655. self._check_double_roundtrip(sp2, tm.assert_panel_equal,
  656. check_panel_type=True)
  657. sp3 = p.to_sparse(fill_value=0)
  658. self._check_double_roundtrip(sp3, tm.assert_panel_equal,
  659. check_panel_type=True)
  660. def test_float_index(self):
  661. # GH #454
  662. index = np.random.randn(10)
  663. s = Series(np.random.randn(10), index=index)
  664. self._check_roundtrip(s, tm.assert_series_equal)
  665. def test_tuple_index(self):
  666. # GH #492
  667. col = np.arange(10)
  668. idx = [(0.,1.), (2., 3.), (4., 5.)]
  669. data = np.random.randn(30).reshape((3, 10))
  670. DF = DataFrame(data, index=idx, columns=col)
  671. self._check_roundtrip(DF, tm.assert_frame_equal)
  672. def test_index_types(self):
  673. values = np.random.randn(2)
  674. func = lambda l, r : tm.assert_series_equal(l, r, True, True, True)
  675. ser = Series(values, [0, 'y'])
  676. self._check_roundtrip(ser, func)
  677. ser = Series(values, [datetime.today(), 0])
  678. self._check_roundtrip(ser, func)
  679. ser = Series(values, ['y', 0])
  680. self._check_roundtrip(ser, func)
  681. from datetime import date
  682. ser = Series(values, [date.today(), 'a'])
  683. self._check_roundtrip(ser, func)
  684. ser = Series(values, [1.23, 'b'])
  685. self._check_roundtrip(ser, func)
  686. ser = Series(values, [1, 1.53])
  687. self._check_roundtrip(ser, func)
  688. ser = Series(values, [1, 5])
  689. self._check_roundtrip(ser, func)
  690. ser = Series(values, [datetime(2012, 1, 1), datetime(2012, 1, 2)])
  691. self._check_roundtrip(ser, func)
  692. def test_timeseries_preepoch(self):
  693. if sys.version_info[0] == 2 and sys.version_info[1] < 7:
  694. raise nose.SkipTest
  695. dr = bdate_range('1/1/1940', '1/1/1960')
  696. ts = Series(np.random.randn(len(dr)), index=dr)
  697. try:
  698. self._check_roundtrip(ts, tm.assert_series_equal)
  699. except OverflowError:
  700. raise nose.SkipTest('known failer on some windows platforms')
  701. def test_frame(self):
  702. df = tm.makeDataFrame()
  703. # put in some random NAs
  704. df.values[0, 0] = np.nan
  705. df.values[5, 3] = np.nan
  706. self._check_roundtrip_table(df, tm.assert_frame_equal)
  707. self._check_roundtrip(df, tm.assert_frame_equal)
  708. self._check_roundtrip_table(df, tm.assert_frame_equal,
  709. compression=True)
  710. self._check_roundtrip(df, tm.assert_frame_equal,
  711. compression=True)
  712. tdf = tm.makeTimeDataFrame()
  713. self._check_roundtrip(tdf, tm.assert_frame_equal)
  714. self._check_roundtrip(tdf, tm.assert_frame_equal,
  715. compression=True)
  716. # not consolidated
  717. df['foo'] = np.random.randn(len(df))
  718. self.store['df'] = df
  719. recons = self.store['df']
  720. self.assert_(recons._data.is_consolidated())
  721. # empty
  722. self._check_roundtrip(df[:0], tm.assert_frame_equal)
  723. def test_empty_series_frame(self):
  724. s0 = Series()
  725. s1 = Series(name='myseries')
  726. df0 = DataFrame()
  727. df1 = DataFrame(index=['a', 'b', 'c'])
  728. df2 = DataFrame(columns=['d', 'e', 'f'])
  729. self._check_roundtrip(s0, tm.assert_series_equal)
  730. self._check_roundtrip(s1, tm.assert_series_equal)
  731. self._check_roundtrip(df0, tm.assert_frame_equal)
  732. self._check_roundtrip(df1, tm.assert_frame_equal)
  733. self._check_roundtrip(df2, tm.assert_frame_equal)
  734. def test_can_serialize_dates(self):
  735. rng = [x.date() for x in bdate_range('1/1/2000', '1/30/2000')]
  736. frame = DataFrame(np.random.randn(len(rng), 4), index=rng)
  737. self._check_roundtrip(frame, tm.assert_frame_equal)
  738. def test_timezones(self):
  739. rng = date_range('1/1/2000', '1/30/2000', tz='US/Eastern')
  740. frame = DataFrame(np.random.randn(len(rng), 4), index=rng)
  741. try:
  742. store = HDFStore(self.scratchpath)
  743. store['frame'] = frame
  744. recons = store['frame']
  745. self.assert_(recons.index.equals(rng))
  746. self.assertEquals(rng.tz, recons.index.tz)
  747. finally:
  748. store.close()
  749. os.remove(self.scratchpath)
  750. def test_fixed_offset_tz(self):
  751. rng = date_range('1/1/2000 00:00:00-07:00', '1/30/2000 00:00:00-07:00')
  752. frame = DataFrame(np.random.randn(len(rng), 4), index=rng)
  753. try:
  754. store = HDFStore(self.scratchpath)
  755. store['frame'] = frame
  756. recons = store['frame']
  757. self.assert_(recons.index.equals(rng))
  758. self.assertEquals(rng.tz, recons.index.tz)
  759. finally:
  760. store.close()
  761. os.remove(self.scratchpath)
  762. def test_store_hierarchical(self):
  763. index = MultiIndex(levels=[['foo', 'bar', 'baz', 'qux'],
  764. ['one', 'two', 'three']],
  765. labels=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3],
  766. [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],
  767. names=['foo', 'bar'])
  768. frame = DataFrame(np.random.randn(10, 3), index=index,
  769. columns=['A', 'B', 'C'])
  770. self._check_roundtrip(frame, tm.assert_frame_equal)
  771. self._check_roundtrip(frame.T, tm.assert_frame_equal)
  772. self._check_roundtrip(frame['A'], tm.assert_series_equal)
  773. # check that the names are stored
  774. try:
  775. store = HDFStore(self.scratchpath)
  776. store['frame'] = frame
  777. recons = store['frame']
  778. assert(recons.index.names == ['foo', 'bar'])
  779. finally:
  780. store.close()
  781. os.remove(self.scratchpath)
  782. def test_store_index_name(self):
  783. df = tm.makeDataFrame()
  784. df.index.name = 'foo'
  785. try:
  786. store = HDFStore(self.scratchpath)
  787. store['frame'] = df
  788. recons = store['frame']
  789. assert(recons.index.name == 'foo')
  790. finally:
  791. store.close()
  792. os.remove(self.scratchpath)
  793. def test_store_series_name(self):
  794. df = tm.makeDataFrame()
  795. series = df['A']
  796. try:
  797. store = HDFStore(self.scratchpath)
  798. store['series'] = series
  799. recons = store['series']
  800. assert(recons.name == 'A')
  801. finally:
  802. store.close()
  803. os.remove(self.scratchpath)
  804. def test_store_mixed(self):
  805. def _make_one():
  806. df = tm.makeDataFrame()
  807. df['obj1'] = 'foo'
  808. df['obj2'] = 'bar'
  809. df['bool1'] = df['A'] > 0
  810. df['bool2'] = df['B'] > 0
  811. df['int1'] = 1
  812. df['int2'] = 2
  813. return df.consolidate()
  814. df1 = _make_one()
  815. df2 = _make_one()
  816. self._check_roundtrip(df1, tm.assert_frame_equal)
  817. self._check_roundtrip(df2, tm.assert_frame_equal)
  818. self.store['obj'] = df1
  819. tm.assert_frame_equal(self.store['obj'], df1)
  820. self.store['obj'] = df2
  821. tm.assert_frame_equal(self.store['obj'], df2)
  822. # check that can store Series of all of these types
  823. self._check_roundtrip(df1['obj1'], tm.assert_series_equal)
  824. self._check_roundtrip(df1['bool1'], tm.assert_series_equal)
  825. self._check_roundtrip(df1['int1'], tm.assert_series_equal)
  826. # try with compression
  827. self._check_roundtrip(df1['obj1'], tm.assert_series_equal,
  828. compression=True)
  829. self._check_roundtrip(df1['bool1'], tm.assert_series_equal,
  830. compression=True)
  831. self._check_roundtrip(df1['int1'], tm.assert_series_equal,
  832. compression=True)
  833. self._check_roundtrip(df1, tm.assert_frame_equal,
  834. compression=True)
  835. def test_wide(self):
  836. wp = tm.makePanel()
  837. self._check_roundtrip(wp, tm.assert_panel_equal)
  838. def test_wide_table(self):
  839. wp = tm.makePanel()
  840. self._check_roundtrip_table(wp, tm.assert_panel_equal)
  841. def test_wide_table_dups(self):
  842. wp = tm.makePanel()
  843. try:
  844. store = HDFStore(self.scratchpath)
  845. store._quiet = True
  846. store.put('panel', wp, table=True)
  847. store.put('panel', wp, table=True, append=True)
  848. recons = store['panel']
  849. tm.assert_panel_equal(recons, wp)
  850. finally:
  851. store.close()
  852. os.remove(self.scratchpath)
  853. def test_long(self):
  854. def _check(left, right):
  855. tm.assert_panel_equal(left.to_panel(), right.to_panel())
  856. wp = tm.makePanel()
  857. self._check_roundtrip(wp.to_frame(), _check)
  858. # empty
  859. # self._check_roundtrip(wp.to_frame()[:0], _check)
  860. def test_longpanel(self):
  861. pass
  862. def test_overwrite_node(self):
  863. self.store['a'] = tm.makeTimeDataFrame()
  864. ts = tm.makeTimeSeries()
  865. self.store['a'] = ts
  866. tm.assert_series_equal(self.store['a'], ts)
  867. def test_select(self):
  868. wp = tm.makePanel()
  869. # put/select ok
  870. self.store.remove('wp')
  871. self.store.put('wp', wp, table=True)
  872. self.store.select('wp')
  873. # non-table ok (where = None)
  874. self.store.remove('wp')
  875. self.store.put('wp2', wp, table=False)
  876. self.store.select('wp2')
  877. # selection on the non-indexable with a large number of columns
  878. wp = Panel(np.random.randn(100, 100, 100), items = [ 'Item%03d' % i for i in xrange(100) ],
  879. major_axis=date_range('1/1/2000', periods=100), minor_axis = [ 'E%03d' % i for i in xrange(100) ])
  880. self.store.remove('wp')
  881. self.store.append('wp', wp)
  882. items = [ 'Item%03d' % i for i in xrange(80) ]
  883. result = self.store.select('wp', Term('items', items))
  884. expected = wp.reindex(items = items)
  885. tm.assert_panel_equal(expected, result)
  886. # selectin non-table with a where
  887. #self.assertRaises(Exception, self.store.select,
  888. # 'wp2', ('column', ['A', 'D']))
  889. def test_panel_select(self):
  890. wp = tm.makePanel()
  891. self.store.put('wp', wp, table=True)
  892. date = wp.major_axis[len(wp.major_axis) // 2]
  893. crit1 = ('major_axis','>=',date)
  894. crit2 = ('minor_axis', '=', ['A', 'D'])
  895. result = self.store.select('wp', [crit1, crit2])
  896. expected = wp.truncate(before=date).reindex(minor=['A', 'D'])
  897. tm.assert_panel_equal(result, expected)
  898. result = self.store.select('wp', [ 'major_axis>=20000124', ('minor_axis', '=', ['A','B']) ])
  899. expected = wp.truncate(before='20000124').reindex(minor=['A', 'B'])
  900. tm.assert_panel_equal(result, expected)
  901. def test_frame_select(self):
  902. df = tm.makeTimeDataFrame()
  903. self.store.put('frame', df, table=True)
  904. date = df.index[len(df) // 2]
  905. crit1 = ('index','>=',date)
  906. crit2 = ('columns',['A', 'D'])
  907. crit3 = ('columns','A')
  908. result = self.store.select('frame', [crit1, crit2])
  909. expected = df.ix[date:, ['A', 'D']]
  910. tm.assert_frame_equal(result, expected)
  911. result = self.store.select('frame', [crit3])
  912. expected = df.ix[:, ['A']]
  913. tm.assert_frame_equal(result, expected)
  914. # other indicies for a frame
  915. # integer
  916. df = DataFrame(dict(A = np.random.rand(20), B = np.random.rand(20)))
  917. self.store.append('df_int', df)
  918. self.store.select('df_int', [ Term("index<10"), Term("columns", "=", ["A"]) ])
  919. df = DataFrame(dict(A = np.random.rand(20), B = np.random.rand(20), index = np.arange(20,dtype='f8')))
  920. self.store.append('df_float', df)
  921. self.store.select('df_float', [ Term("index<10.0"), Term("columns", "=", ["A"]) ])
  922. # can't select if not written as table
  923. #self.store['frame'] = df
  924. #self.assertRaises(Exception, self.store.select,
  925. # 'frame', [crit1, crit2])
  926. def test_select_filter_corner(self):
  927. df = DataFrame(np.random.randn(50, 100))
  928. df.index = ['%.3d' % c for c in df.index]
  929. df.columns = ['%.3d' % c for c in df.columns]
  930. self.store.put('frame', df, table=True)
  931. crit = Term('columns', df.columns[:75])
  932. result = self.store.select('frame', [crit])
  933. tm.assert_frame_equal(result, df.ix[:, df.columns[:75]])
  934. def _check_roundtrip(self, obj, comparator, compression=False, **kwargs):
  935. options = {}
  936. if compression:
  937. options['complib'] = _default_compressor
  938. store = HDFStore(self.scratchpath, 'w', **options)
  939. try:
  940. store['obj'] = obj
  941. retrieved = store['obj']
  942. comparator(retrieved, obj, **kwargs)
  943. finally:
  944. store.close()
  945. os.remove(self.scratchpath)
  946. def _check_double_roundtrip(self, obj, comparator, compression=False,
  947. **kwargs):
  948. options = {}
  949. if compression:
  950. options['complib'] = _default_compressor
  951. store = HDFStore(self.scratchpath, 'w', **options)
  952. try:
  953. store['obj'] = obj
  954. retrieved = store['obj']
  955. comparator(retrieved, obj, **kwargs)
  956. store['obj'] = retrieved
  957. again = store['obj']
  958. comparator(again, obj, **kwargs)
  959. finally:
  960. store.close()
  961. os.remove(self.scratchpath)
  962. def _check_roundtrip_table(self, obj, comparator, compression=False):
  963. options = {}
  964. if compression:
  965. options['complib'] = _default_compressor
  966. store = HDFStore(self.scratchpath, 'w', **options)
  967. try:
  968. store.put('obj', obj, table=True)
  969. retrieved = store['obj']
  970. # sorted_obj = _test_sort(obj)
  971. comparator(retrieved, obj)
  972. finally:
  973. store.close()
  974. os.remove(self.scratchpath)
  975. def test_legacy_read(self):
  976. pth = curpath()
  977. store = HDFStore(os.path.join(pth, 'legacy.h5'), 'r')
  978. store['a']
  979. store['b']
  980. store['c']
  981. store['d']
  982. store.close()
  983. def test_legacy_table_read(self):
  984. # legacy table types
  985. pth = curpath()
  986. store = HDFStore(os.path.join(pth, 'legacy_table.h5'), 'r')
  987. store.select('df1')
  988. store.select('df2')
  989. store.select('wp1')
  990. # force the frame
  991. store.select('df2', typ = 'legacy_frame')
  992. # old version (this still throws an exception though)
  993. import warnings
  994. warnings.filterwarnings('ignore', category=IncompatibilityWarning)
  995. self.assertRaises(Exception, store.select, 'wp1', Term('minor_axis','=','B'))
  996. warnings.filterwarnings('always', category=IncompatibilityWarning)
  997. store.close()
  998. def test_legacy_table_write(self):
  999. # legacy table types
  1000. pth = curpath()
  1001. df = tm.makeDataFrame()
  1002. wp = tm.makePanel()
  1003. store = HDFStore(os.path.join(pth, 'legacy_table.h5'), 'a')
  1004. self.assertRaises(Exception, store.append, 'df1', df)
  1005. self.assertRaises(Exception, store.append, 'wp1', wp)
  1006. store.close()
  1007. def test_store_datetime_fractional_secs(self):
  1008. dt = datetime(2012, 1, 2, 3, 4, 5, 123456)
  1009. series = Series([0], [dt])
  1010. self.store['a'] = series
  1011. self.assertEquals(self.store['a'].index[0], dt)
  1012. def test_tseries_indices_series(self):
  1013. idx = tm.makeDateIndex(10)
  1014. ser = Series(np.random.randn(len(idx)), idx)
  1015. self.store['a'] = ser
  1016. result = self.store['a']
  1017. assert_series_equal(result, ser)
  1018. self.assertEquals(type(result.index), type(ser.index))
  1019. self.assertEquals(result.index.freq, ser.index.freq)
  1020. idx = tm.makePeriodIndex(10)
  1021. ser = Series(np.random.randn(len(idx)), idx)
  1022. self.store['a'] = ser
  1023. result = self.store['a']
  1024. assert_series_equal(result, ser)
  1025. self.assertEquals(type(result.index), type(ser.index))
  1026. self.assertEquals(result.index.freq, ser.index.freq)
  1027. def test_tseries_indices_frame(self):
  1028. idx = tm.makeDateIndex(10)
  1029. df = DataFrame(np.random.randn(len(idx), 3), index=idx)
  1030. self.store['a'] = df
  1031. result = self.store['a']
  1032. assert_frame_equal(result, df)
  1033. self.assertEquals(type(result.index), type(df.index))
  1034. self.assertEquals(result.index.freq, df.index.freq)
  1035. idx = tm.makePeriodIndex(10)
  1036. df = DataFrame(np.random.randn(len(idx), 3), idx)
  1037. self.store['a'] = df
  1038. result = self.store['a']
  1039. assert_frame_equal(result, df)
  1040. self.assertEquals(type(result.index), type(df.index))
  1041. self.assertEquals(result.index.freq, df.index.freq)
  1042. def test_unicode_index(self):
  1043. unicode_values = [u'\u03c3', u'\u03c3\u03c3']
  1044. s = Series(np.random.randn(len(unicode_values)), unicode_values)
  1045. self._check_roundtrip(s, tm.assert_series_equal)
  1046. def test_store_datetime_mixed(self):
  1047. df = DataFrame({'a': [1,2,3], 'b': [1.,2.,3.], 'c': ['a', 'b', 'c']})
  1048. ts = tm.makeTimeSeries()
  1049. df['d'] = ts.index[:3]
  1050. self._check_roundtrip(df, tm.assert_frame_equal)
  1051. def test_cant_write_multiindex_table(self):
  1052. # for now, #1848
  1053. df = DataFrame(np.random.randn(10, 4),
  1054. index=[np.arange(5).repeat(2),
  1055. np.tile(np.arange(2), 5)])
  1056. self.assertRaises(Exception, self.store.put, 'foo', df, table=True)
  1057. def curpath():
  1058. pth, _ = os.path.split(os.path.abspath(__file__))
  1059. return pth
  1060. def _test_sort(obj):
  1061. if isinstance(obj, DataFrame):
  1062. return obj.reindex(sorted(obj.index))
  1063. elif isinstance(obj, Panel):
  1064. return obj.reindex(major=sorted(obj.major_axis))
  1065. else:
  1066. raise ValueError('type not supported here')
  1067. if __name__ == '__main__':
  1068. import nose
  1069. nose.runmodule(argv=[__file__,'-vvs','-x','--pdb', '--pdb-failure'],
  1070. exit=False)