PageRenderTime 22ms CodeModel.GetById 26ms RepoModel.GetById 0ms app.codeStats 0ms

/mat2nc.py

https://gitlab.com/pkormos/phase_python
Python | 193 lines | 36 code | 26 blank | 131 comment | 0 complexity | 2292cb31e8d95c88d3e277d076a93c8a MD5 | raw file
    

  1. '''
    2. 

  2. bring_in_ml_files -- this program will take the corrected weather data in the .mat files and pull it into netcdf formats
    3. 

  3. 

  4. does precip station data
    5. 

  5. rh station data (which is then modified and resaved)
    6. 

  6. precipitation totals
    7. 

  7. 

  8. @author:     patrick kormos
    9. 

  9. @contact:    patrick.kormos@ars.usda.gov
    10. 

  10. '''
    11. 

    12. 

  12. import netCDF4 as nc
    13. 

  13. import numpy as np
    14. 

  14. from pandas import DataFrame, date_range
    15. 

  15. import h5py
    16. 

  16. 

  17. 

  18. # #### THIS CODE CHUNK WILL CONVERT PRECIPITATION STATION DATA .MAT FILE TO .NC FILE ####
    19. 

  19. # pmat = h5py.File('/Users/pkormos/rcew_met_dist/met/latest_data/rcew_ppt_pk.mat') # connect to .mat database
    20. 

  20. # p_list = np.array(['rc-047', 'rc-055', 'rc-059', 'rc-074', 'rc-075', 'rc-076', 'rc-078', 'rc-083', # manually bring in column headers
    21. 

  21. #     'rc-088', 'rc-095b', 'rc-097', 'rc-106', 'rc-114', 'rc-119', 'rc-128', 'rc.fl-057',
    22. 

  22. #     'rc.lsc-127', 'rc.mc-043041', 'rc.mc-053', 'rc.mc-054', 'rc.mc-061', 'rc.mc-072',
    23. 

  23. #     'rc.ng-098c', 'rc.ng-108', 'rc.sc-012', 'rc.sc-015', 'rc.sc-023', 'rc.sc-024',
    24. 

  24. #     'rc.sc-031','rc.sc-045', 'rc.sc.mp-033', 'rc.sum-049', 'rc.tg-116c','rc.tg-126',
    25. 

  25. #     'rc.tg-145', 'rc.tg-147', 'rc.tg-155', 'rc.tg-156', 'rc.tg-165', 'rc.tg-167',
    26. 

  26. #     'rc.tg-174', 'rc.tg.dc-144', 'rc.tg.dc-154', 'rc.tg.dc-163', 'rc.tg.dc-163',
    27. 

  27. #     'rc.tg.dc.jd-124', 'rc.tg.dc.jd-124b', 'rc.tg.dc.jd-125', 'rc.tg.rme-166b',
    28. 

  28. #     'rc.tg.rme-176', 'rc.tg.rme-rmsp', 'rc.tg.rmw-166x94','rc.usc-138031', 
    29. 

  29. #     'rc.usc-138044', 'rc.usc-d03','rc.usc-j10', 'rc.usc-l21'])
    30. 

  30. # ppt = DataFrame(np.transpose(pmat.get("ppt_corrected")), index=date_range('1962-1-1', '2014-12-31 23:00', freq='H'), columns = p_list) # make pandas dataset
    31. 

  31. # ppt = ppt.replace('NaN',-9999)
    32. 

  32. #  
    33. 

  33. #  
    34. 

  34. # # create the ppt netcdf file.
    35. 

  35. # pnc = '/Volumes/bunnyhill/HUMIDITY_WORK_2016/rcew_ppt_pk.nc' # netcdf file name
    36. 

  36. # pn = nc.Dataset(pnc, 'w',format="NETCDF4",clobber=True)            # create precip ncdf file
    37. 

  37. # time = pn.createDimension("time", None)               # create time dimension
    38. 

  38. # stns = pn.createDimension("stations",np.size(p_list)) # create stations dimension
    39. 

  39. # tvar = pn.createVariable("time","int",("time"))
    40. 

  40. # tvar[:] = np.arange(0,np.shape(ppt)[0])
    41. 

  41. # tvar.units = "hours since 1962-01-01 00:00:00 -7:00"
    42. 

  42. # tvar.long_name = "time"
    43. 

  43. # stnn = pn.createVariable("stations","int",("stations"))
    44. 

  44. # for i in np.arange(0,np.shape(p_list)[0]):
    45. 

  45. #     setattr(pn.variables['stations'], 'station_%s'%i,p_list[i])
    46. 

  46. # stnn[:] = np.arange(0,np.shape(p_list)[0])
    47. 

  47. # pvar = pn.createVariable("precipitation","f4",("time","stations",),fill_value=-9999)
    48. 

  48. # pvar.long_name = "corrected precipitation depth per hour time step"
    49. 

  49. # pvar.units = "mm"
    50. 

  50. # pvar[:] = ppt.as_matrix()
    51. 

  51. # pn.close()
    52. 

  52. 

  53. ####################################################################################################################################
    54. 

  54. 

  55. # # rh has been modified by plot_and_correct_humitiyxxx.py. if you uncomment and rerun this code, you must rerun these scripts.
    56. 

  56. 

  57. #### THIS CODE CHUNK WILL CONVERT RELATIVE HUMIDITY MAT FILE TO A .NC FILE ####
    58. 

  58. rmat = h5py.File('/Users/pkormos/rcew_met_dist/met/latest_data/rcew_rh_pk.mat') # connect to .mat database
    59. 

  59. r_list = np.array(['rc-076','rc-095b','rc-128','rc.lsc-127','rc.sc-012','rc.sc-031',
    60. 

  60.     'rc.tg-145','rc.tg-167','rc.tg-167b','rc.tg-174','rc.tg.dc-144',
    61. 

  61.     'rc.tg.dc-163',
    62. 

  62.     'rc.tg.dc.jd-124',
    63. 

  63.     'rc.tg.dc.jd-124b',
    64. 

  64.     'rc.tg.dc.jd-125',
    65. 

  65.     'rc.tg.rme-166b',
    66. 

  66.     'rc.tg.rme-176',
    67. 

  67.     'rc.tg.rme-rmsp',
    68. 

  68.     'rc.usc-d03',
    69. 

  69.     'rc.usc-j10',
    70. 

  70.     'rc.usc-l21',
    71. 

  71.     'rc-076_tf',
    72. 

  72.     'rc-128_tf',
    73. 

  73.     'rc.lsc-127_tf',
    74. 

  74.     'rc.tg.rme-176_tf'])
    75. 

  75. rh = DataFrame(np.transpose(rmat.get("rh")), index=date_range('18-Jun-1981 11:00:00', '01-Oct-2014', freq='H'), columns = r_list) # make pandas dataset
    76. 

  76. rh = rh.replace('NaN',-9999)
    77. 

  77. rnc = '/Volumes/bunnyhill/HUMIDITY_WORK_2016/rcew_rh_pk.nc' # netcdf file name
    78. 

  78. # rn = nc.Dataset(rnc, 'w',format="NETCDF4",clobber=True)            # create rh ncdf file
    79. 

  79. # time = rn.createDimension("time", None)               # create time dimension
    80. 

  80. # stns = rn.createDimension("stations",np.size(r_list)) # create stations dimension
    81. 

  81. # tvar = rn.createVariable("time","int",("time"))
    82. 

  82. # tvar[:] = np.arange(0,np.shape(rh)[0])
    83. 

  83. # tvar.units = "hours since 1981-06-18 11:00:00 -7:00"
    84. 

  84. # tvar.long_name = "time"
    85. 

  85. # stnn = rn.createVariable("stations","int",("stations"))
    86. 

  86. # stnn[:] = np.arange(0,np.shape(r_list)[0])
    87. 

  87. # for i in np.arange(0,np.shape(r_list)[0]):
    88. 

  88. #     setattr(rn.variables['stations'], 'station_%s'%i,r_list[i])
    89. 

  89. # rvar = rn.createVariable("relative_humidity","f4",("time","stations",),fill_value=-9999)
    90. 

  90. # rvar.long_name = "relative humidity"
    91. 

  91. # rvar.units = "decimal percent"
    92. 

  92. # rvar[:] = rh.as_matrix()
    93. 

  93. # rn.close()
    94. 

  94. rn = nc.Dataset(rnc, 'r+')  # open rh ncdf file
    95. 

  95. rn_rh = rn.variables['relative_humidity']
    96. 

  96. rn_rh[:,np.squeeze(np.argwhere(r_list=='rc-128'))] = rh['rc-128'].as_matrix()
    97. 

  97. rn.close()
    98. 

  98. 

  99.  
    100. 

  100. # # append station xyz onto rh file
    101. 

  101. # rmat = h5py.File('/Users/pkormos/rcew_met_dist/met/latest_data/rcew_rh_pk.mat') # connect to .mat database
    102. 

  102. coords = rmat.get("rh_stn_xyz")
    103. 

  103. # rnc = '/Volumes/bunnyhill/HUMIDITY_WORK_2016/rcew_rh_pk.nc' # netcdf file name
    104. 

  104. # rn = nc.Dataset(rnc, 'r+',format="NETCDF4")            # create precip ncdf file
    105. 

  105. xyzdim = rn.createDimension("easting_northing_elev",3) # create stations dimension
    106. 

  106. xyzvar = rn.createVariable("easting_northing_elev","float",("stations","easting_northing_elev"))
    107. 

  107. xyzvar[:] = np.transpose(coords[:])
    108. 

  108. xyzvar.station_coordinate_0 = "easting (meters)"
    109. 

  109. xyzvar.station_coordinate_1 = "northing (meters)"
    110. 

  110. xyzvar.Coordinate_System = "UTM_NAD83_Zone11"
    111. 

  111. rn.close()
    112. 

  112. 

  113. 

  114. # ####################################################################################################################################
    115. 

  115. # ####### CODE TO CONVERT WATER YEAR TOTAL PRECIPTIATION FROM .MAT FILE TO NETCDF FILE ###############################################
    116. 

  116. # ####################################################################################################################################
    117. 

  117. # wyoi = range(1984,2015)
    118. 

  118. # ptmat = h5py.File('/Users/pkormos/rcew_met_dist/snow_data/phase_analysis/ppt_wytot.mat') # connect to .mat database
    119. 

  119. # ppt = np.transpose(ptmat.get("ppt_wytot"),(0,2,1))
    120. 

  120. 

  121. 

  122. # ppt_file = "/Volumes/LaCie/precip_cf/precip_wy1984.nc"
    123. 

  123. # p_ncid = nc.Dataset(ppt_file,'r')                      # open ncdf file with correct x and y information
    124. 

  124. 

  125. # ppt_tot_file = nc.Dataset("/Volumes/megadozer/CZO/data_products/rcew_ppt_wytot.nc",'w')
    126. 

  126. # ppt_tot_file.createDimension('time',size=None) # make an unlimited dimension
    127. 

  127. 

  128. # ydim = p_ncid.dimensions['y']
    129. 

  129. # ppt_tot_file.createDimension(ydim.name,len(ydim))
    130. 

  130. # xdim = p_ncid.dimensions['x']
    131. 

  131. # ppt_tot_file.createDimension(xdim.name,len(xdim))
    132. 

  132. 

  133. 

  134. # wyvar = ppt_tot_file.createVariable("time","i","time")
    135. 

  135. # wyvar[:] = wyoi
    136. 

  136. 

  137. # px = p_ncid.variables['x']
    138. 

  138. # x  = ppt_tot_file.createVariable(px.name,px.datatype,px.dimensions) # make same xvar
    139. 

  139. # x[:] = px[:] # put x coords in there
    140. 

  140. 

  141. # py = p_ncid.variables['y']
    142. 

  142. # y  = ppt_tot_file.createVariable(py.name,py.datatype,py.dimensions) # make same xvar
    143. 

  143. # y[:] = py[:] # put x coords in there
    144. 

  144. 

  145. # wytot_var = ppt_tot_file.createVariable('ppt_wytot','f4',('time','y','x')) # create vp variable
    146. 

  146. # wytot_var[:] = ppt[:,:,:]
    147. 

  147. 

  148. # p_ncid.close()
    149. 

  149. # ppt_tot_file.close()
    150. 

  150.   
    151. 

  151. #  # could improve this sometime to make it more netcdf complient
    152. 

  152. 

  153. ####################################################################################################################################
    154. 

  154. ####### CODE TO CONVERT WINTER WATER YEAR TOTAL PRECIPTIATION FROM .MAT FILE TO NETCDF FILE ########################################
    155. 

  155. ####################################################################################################################################
    156. 

  156. # wyoi = range(1984,2015)
    157. 

  157. # ptmat = h5py.File('/Users/pkormos/rcew_met_dist/snow_data/phase_analysis/ppt_wtot.mat') # connect to .mat database
    158. 

  158. # ppt = np.transpose(ptmat.get("wppt_total"),(0,2,1))
    159. 

  159. # 
    160. 

  160. # 
    161. 

  161. # ppt_file = "/Volumes/LaCie/precip_cf/precip_wy1984.nc"
    162. 

  162. # p_ncid = nc.Dataset(ppt_file,'r')                      # open ncdf file with correct x and y information
    163. 

  163. # 
    164. 

  164. # ppt_tot_file = nc.Dataset("/Volumes/megadozer/CZO/data_products/rcew_ppt_winter_tot.nc",'w')
    165. 

  165. # ppt_tot_file.createDimension('time',size=None) # make an unlimited dimension
    166. 

  166. # 
    167. 

  167. # ydim = p_ncid.dimensions['y']
    168. 

  168. # ppt_tot_file.createDimension(ydim.name,len(ydim))
    169. 

  169. # xdim = p_ncid.dimensions['x']
    170. 

  170. # ppt_tot_file.createDimension(xdim.name,len(xdim))
    171. 

  171. # 
    172. 

  172. # 
    173. 

  173. # wyvar = ppt_tot_file.createVariable("time","i","time")
    174. 

  174. # wyvar[:] = wyoi
    175. 

  175. # 
    176. 

  176. # px = p_ncid.variables['x']
    177. 

  177. # x  = ppt_tot_file.createVariable(px.name,px.datatype,px.dimensions) # make same xvar
    178. 

  178. # x[:] = px[:] # put x coords in there
    179. 

  179. # 
    180. 

  180. # py = p_ncid.variables['y']
    181. 

  181. # y  = ppt_tot_file.createVariable(py.name,py.datatype,py.dimensions) # make same xvar
    182. 

  182. # y[:] = py[:] # put x coords in there
    183. 

  183. # 
    184. 

  184. # wytot_var = ppt_tot_file.createVariable('ppt_winter_tot','f4',('time','y','x')) # create vp variable
    185. 

  185. # wytot_var[:] = ppt[:,:,:]
    186. 

  186. # 
    187. 

  187. # p_ncid.close()
    188. 

  188. # ppt_tot_file.close()
    189. 

  189.   
    190. 

  190.  # could improve this sometime to make it more netcdf complient
    191. 

  191. 

  192. 

  193. 

  194.