PageRenderTime 38ms CodeModel.GetById 9ms RepoModel.GetById 0ms app.codeStats 0ms

/accelometer_metadata_creator.py

https://gitlab.com/heavelock/metadata_creator
Python | 198 lines | 172 code | 15 blank | 11 comment | 10 complexity | 8964cda1aa3179553d4cddbc9610e47b MD5 | raw file
  1. from bs4 import BeautifulSoup
  2. import scipy.io
  3. import os
  4. import obspy
  5. from datetime import datetime, timedelta
  6. # Parsowanie pliku xml z metadanymi
  7. def parse_event_names(file):
  8. soup = BeautifulSoup(open(file), 'lxml-xml')
  9. parsed_meta = {}
  10. for ev in soup.find_all('event'):
  11. parsed_meta[str(ev.file.contents[0])] = ev['id']
  12. return parsed_meta
  13. # parsowanie katalogu
  14. def parse_catalog(file):
  15. mat = scipy.io.loadmat(file)
  16. for key in mat.keys():
  17. if key[0:2] != '__':
  18. break
  19. cata_len = len(mat[key][0][0][2])
  20. catalog = {}
  21. for subcat in mat[key][0]:
  22. catalog[subcat[0][0].lower()] = [subcat[2][i][0] for i in range(cata_len)]
  23. return catalog
  24. def write_metadata_file(meta_file, description, starttime, endtime, event_id):
  25. # Zapis do pliku
  26. meta_file.write('<metadata>' + "\n")
  27. meta_file.write(' <item name="episodeName" isRequired="true" minLength="10"/>' + "\n")
  28. meta_file.write(' <item name="episodeCode" isRequired="true" minLength="2"/>' + "\n")
  29. meta_file.write(
  30. ' <item name="episodeOwner" isRequired="true" enum="[&quot;IGF PAS&quot;, &quot;KWSA&quot;, &quot;GIG&quot;,&quot;GFZ&quot;, &quot;VAST&quot;]"/>' + "\n")
  31. meta_file.write(description + "\n")
  32. meta_file.write(' <item name="text" isRequired="true"/>' + "\n")
  33. meta_file.write(
  34. ' <item name="country" isRequired="true" enum="[&quot;Poland&quot;, &quot;Germany&quot;, &quot;Vietnam&quot;]"/>' + "\n")
  35. meta_file.write(
  36. ' <item name="region" enum="[&quot;Upper Silesia&quot;, &quot;Gross Schoenebeck&quot;, &quot;Legnica-Glogow Copper District&quot;, &quot;Niedzica&quot;, &quot;Quang Nam Province&quot;]"/>' + "\n")
  37. meta_file.write(
  38. ' <item name="positionType" isRequired="true" enum="[&quot;point&quot;, &quot;polygon&quot;]"/>' + "\n")
  39. meta_file.write(
  40. ' <item name="coordinateSystem" isRequired="true" enum="[&quot;LOCAL:POL001&quot;, &quot;WGS-84&quot;]"/>' + "\n")
  41. meta_file.write(' <item name="longitude" isRequired="true"/>' + "\n")
  42. meta_file.write(' <item name="latitude" isRequired="true"/>' + "\n")
  43. meta_file.write(starttime + "\n")
  44. meta_file.write(endtime + "\n")
  45. meta_file.write(event_id + "\n")
  46. meta_file.write(' <item name="itemType" readOnly="true"/>' + "\n")
  47. meta_file.write(' <item name="name" readOnly="true"/>' + "\n")
  48. meta_file.write(' <item name="path" readOnly="true"/>' + "\n")
  49. meta_file.write(
  50. ' <item name="dataType" enum="[&quot; &quot;, &quot;Accelerogram Miniseed Signal&quot;, &quot;Accelerogram Seed Signal&quot;, &quot;Air Quality&quot;, &quot;Air Stations&quot;, &quot;Atmospheric Pressure&quot;, &quot;Bottomhole Pressure&quot;, &quot;Catalog&quot;, &quot;Chemical Water Properties - site visit&quot;, &quot;Cumulative Injection&quot;, &quot;Episode Image&quot;, &quot;Episode Logo&quot;, &quot;Episode XML&quot;, &quot;Faults&quot;, &quot;Flowback Bottomhole Pressure&quot;, &quot;Flowback Rate&quot;, &quot;Flowback Volume&quot;, &quot;Geoarea&quot;, &quot;Ground Motion Catalog&quot;, &quot;Ground Motion Network&quot;, &quot;Injection Rate&quot;, &quot;Injection Volume&quot;, &quot;Mining Front Advance&quot;, &quot;Mining Polygon Advance&quot;, &quot;Miniseed Signal&quot;, &quot;Mine Area&quot;, &quot;Physical Water Properties - site visit&quot;, &quot;Physicochemical Water Properties&quot;, &quot;Physicochemical Water Properties - site visit&quot;, &quot;Powerplant Position&quot;, &quot;Proppant Concentration&quot;, &quot;Radon 222 Concentration&quot;, &quot;Ray Tracing Angles&quot;, &quot;Seed Signal&quot;, &quot;Seismic Network&quot;, &quot;Shapefile&quot;, &quot;Shear Wave Velocity&quot;, &quot;Shoreline&quot;, &quot;Steam production&quot;, &quot;Velocity Model&quot;, &quot;Water Lab Analyses&quot;, &quot;Water Level&quot;, &quot;Water Stations&quot;, &quot;Water Volume&quot;, &quot;Well Path&quot;, &quot;Well Position&quot;, &quot;Wellhead Pressure&quot;]"/>' + "\n")
  51. meta_file.write(
  52. ' <item name="impactingFactor" isRequired="true" enum="[&quot;underground mining&quot;, &quot;geothermal energy production&quot;, &quot;reservoir impoundment&quot;]"/>' + "\n")
  53. meta_file.write(
  54. ' <item name="type" enum="[&quot;data relevant for the considered hazards&quot;, &quot;seismic&quot;, &quot;water quality&quot;, &quot;air quality&quot;, &quot;satellite&quot;, &quot;industrial&quot;, &quot;geodata&quot;]"/>' + "\n")
  55. meta_file.write(' <item name="auxiliary" enum="[&quot;1&quot;]"/>' + "\n")
  56. meta_file.write(
  57. ' <item name="allowedDownload" isRequired="true" enum="[&quot;SHEER&quot;, &quot;EPOS-IP&quot;, &quot;all&quot;, &quot;affiliated&quot;]"/>' + "\n")
  58. meta_file.write(
  59. ' <item name="allowedVisibility" isRequired="true" enum="[&quot;SHEER&quot;, &quot;EPOS-IP&quot;, &quot;all&quot;, &quot;affiliated&quot;]"/>' + "\n")
  60. meta_file.write(' <item name="dataPolicy" isRequired="true"/>' + "\n")
  61. meta_file.write('</metadata>' + "\n")
  62. if __name__ == "__main__":
  63. ## Wszystkie inputy za jednym razem
  64. # Podaj sciezke do katalogu roboczego
  65. working_directory = input('Podaj sciezke gdzie znajduja sie wszystkie pliki.' + '\n')
  66. # Sprawdzenie czy podana lokalizacja folderu roboczego istnieje, jesli nie, próba wykonania skryptu w obecnej lokalizacji
  67. if os.path.isdir(working_directory):
  68. # Zmiana folderu roboczego
  69. os.chdir(working_directory)
  70. else:
  71. # Wypisz w konsoli komunikat
  72. print('Podana lokalizacja nie jest poprawna, sprobuje wykonac skrypt w lokalizacji gdzie znajdujde sie plik skryptu.')
  73. # Numer pierwszego sygnalu
  74. first_signal_no = 0
  75. # Numer ostatniego sygnalu
  76. last_signal_no = 9999999
  77. #Prealokacja zmiennej dla katalogu seismo
  78. catalog_name = ''
  79. i = 0
  80. # Petla sprawdzajaca czy podany plik katalogu istnieje
  81. while not os.path.exists(catalog_name):
  82. # Podaj nazwe piku z katalogiem
  83. catalog_name = input('Podaj pelna nazwe pliku z katalogiem.' + '\n')
  84. # Jesli podanie nazwy zostalo pominiete, znajdz plik w folderze roboczym
  85. if catalog_name == '':
  86. catalog_name = [f for f in os.listdir() if f[-12:].lower() == '_catalog.mat'][0]
  87. i += 1
  88. if i == 10:
  89. raise ValueError(
  90. 'Nie udalo Ci sie podac poprawnej nazwy katalogu az 10 razy, w podanym folderze tez nie ma tego pliku.'
  91. ' Sprawdz czy wszystko jest ok.')
  92. # Prealokacja zmiennej dla katalogu seismo
  93. acc_catalog_name = ''
  94. i = 0
  95. # Petla sprawdzajaca czy podany plik katalogu istnieje
  96. while not os.path.exists(acc_catalog_name):
  97. # Podaj nazwe piku z katalogiem
  98. acc_catalog_name = input('Podaj pelna nazwe pliku z ground motion katalogiem.' + '\n')
  99. # Jesli podanie nazwy zostalo pominiete, znajdz plik w folderze roboczym
  100. if acc_catalog_name == '':
  101. acc_catalog_name = [f for f in os.listdir() if f[-14:].lower() == '_gmcatalog.mat'][0]
  102. i += 1
  103. if i == 10:
  104. raise ValueError(
  105. 'Nie udalo Ci sie podac poprawnej nazwy ground motion katalogu az 10 razy, w podanym folderze tez nie ma tego pliku.'
  106. ' Sprawdz czy wszystko jest ok.')
  107. # Podaj nazwe publikacji w CIBIS
  108. try:
  109. cibis_publication_no = int(input('Podaj numer publikacji CIBIS.' + '\n'))
  110. except ValueError:
  111. cibis_publication_no = 1
  112. print('Podana wartosc nie jest liczba. Wartosc ustawiona na 1.')
  113. # Parsowanie catalogu
  114. parsed_catalog = parse_catalog(catalog_name)
  115. acc_parsed_catalog = parse_catalog(acc_catalog_name)
  116. #Prealokacja zmiennej dla typu magnitudy
  117. magnitude_type = None
  118. #Petla sprawdzajaca czy typ magnitudy jest poprawnie wpisanyw
  119. while not magnitude_type in ('ml','mw'):
  120. magnitude_type = input('Wykorzystac mw czy ml? Wpisz mw lub ml.' + '\n')
  121. try:
  122. parsed_catalog[magnitude_type]
  123. except KeyError:
  124. print('W katalogu nie ma magnitudy ', magnitude_type,
  125. '. Spróbuję ustawić magnitudę mw, jeśli program się wysypie to sprawdź katalog. ')
  126. magnitude_type = 'mw'
  127. # Lista plikow seed w katalogu, zawężona do plikow o numerach wiekszych lub rownych numerowi poczatkowemu i
  128. # mniejszych lub rownych numerowi koncowemu
  129. file_list = [f for f in os.listdir() if f[-5:] == '.seed' and
  130. int(f[-11:-5]) >= first_signal_no and
  131. int(f[-11:-5]) <= last_signal_no]
  132. for file in file_list:
  133. # Przeczytaj plik seed
  134. seed_file = obspy.read(file)
  135. # Tworzenie pliku metadanych
  136. metadata_file = open(''.join([file, '_', str(cibis_publication_no), '.metadata']), mode='w')
  137. # Uzyskanie earthquake id z XMLa
  138. earthquake_id = acc_parsed_catalog['eid'][[ i for i,rid in enumerate(acc_parsed_catalog['rid']) if rid[0][:-5] == file[:-5]][0]][0]
  139. # Pobierz magnitude z katalogu
  140. magnitude = parsed_catalog[magnitude_type][parsed_catalog['id'].index(earthquake_id)]
  141. # Pobierz date z katalogu, format matlabowy
  142. matlab_datenum = parsed_catalog['time'][parsed_catalog['id'].index(earthquake_id)]
  143. # Skonwertuj date z formatu matlabowego na pythonowy
  144. date_of_eq = datetime.fromordinal(int(matlab_datenum)) + timedelta(days=matlab_datenum%1) - timedelta(days = 366)
  145. if magnitude_type == 'ml':
  146. description_magnitude_type = ' with ML='
  147. else:
  148. description_magnitude_type = ' with Mw='
  149. # Stworz linijke description
  150. description = ''.join([' <item name="description">','Signal of event (trigger accelerogram) which occurred on ',
  151. date_of_eq.strftime('%Y-%b-%d %X'), description_magnitude_type,
  152. str(magnitude), ' (eventID=', earthquake_id, ')', '</item>'])
  153. # Stworz linijke starttime
  154. starttime = ''.join([' <item name="start">',seed_file[0].stats.starttime.strftime('%Y%m%d%H%M%S.3Z'),'</item>'])
  155. # Stworz linijke endtime
  156. endtime = ''.join([' <item name="end">', seed_file[0].stats.endtime.strftime('%Y%m%d%H%M%S.3Z'), '</item>'])
  157. # Stworz linijke event_id
  158. event_id = ''.join([' <item name="eventID">',earthquake_id,'</item>'])
  159. # Zapisz wszystko do pliku
  160. write_metadata_file(metadata_file,description,starttime,endtime,event_id)
  161. #Zamknij plik
  162. metadata_file.close()
  163. input('Udalo sie! Wcisnij enter aby zamknac okno.')