PageRenderTime 23ms CodeModel.GetById 1ms RepoModel.GetById 0ms app.codeStats 0ms

/indra/lib/python/indra/util/simperf_host_xml_parser.py

https://bitbucket.org/lindenlab/viewer-beta/
Python | 338 lines | 321 code | 6 blank | 11 comment | 8 complexity | 73165cc82ced5d7fc1eca9fb35aad11c MD5 | raw file
Possible License(s): LGPL-2.1
  1. #!/usr/bin/env python
  2. """\
  3. @file simperf_host_xml_parser.py
  4. @brief Digest collector's XML dump and convert to simple dict/list structure
  5. $LicenseInfo:firstyear=2008&license=mit$
  6. Copyright (c) 2008-2009, Linden Research, Inc.
  7. Permission is hereby granted, free of charge, to any person obtaining a copy
  8. of this software and associated documentation files (the "Software"), to deal
  9. in the Software without restriction, including without limitation the rights
  10. to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  11. copies of the Software, and to permit persons to whom the Software is
  12. furnished to do so, subject to the following conditions:
  13. The above copyright notice and this permission notice shall be included in
  14. all copies or substantial portions of the Software.
  15. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18. AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20. OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  21. THE SOFTWARE.
  22. $/LicenseInfo$
  23. """
  24. import sys, os, getopt, time
  25. import simplejson
  26. from xml import sax
  27. def usage():
  28. print "Usage:"
  29. print sys.argv[0] + " [options]"
  30. print " Convert RRD's XML dump to JSON. Script to convert the simperf_host_collector-"
  31. print " generated RRD dump into JSON. Steps include converting selected named"
  32. print " fields from GAUGE type to COUNTER type by computing delta with preceding"
  33. print " values. Top-level named fields are:"
  34. print
  35. print " lastupdate Time (javascript timestamp) of last data sample"
  36. print " step Time in seconds between samples"
  37. print " ds Data specification (name/type) for each column"
  38. print " database Table of data samples, one time step per row"
  39. print
  40. print "Options:"
  41. print " -i, --in Input settings filename. (Default: stdin)"
  42. print " -o, --out Output settings filename. (Default: stdout)"
  43. print " -h, --help Print this message and exit."
  44. print
  45. print "Example: %s -i rrddump.xml -o rrddump.json" % sys.argv[0]
  46. print
  47. print "Interfaces:"
  48. print " class SimPerfHostXMLParser() # SAX content handler"
  49. print " def simperf_host_xml_fixup(parser) # post-parse value fixup"
  50. class SimPerfHostXMLParser(sax.handler.ContentHandler):
  51. def __init__(self):
  52. pass
  53. def startDocument(self):
  54. self.rrd_last_update = 0 # public
  55. self.rrd_step = 0 # public
  56. self.rrd_ds = [] # public
  57. self.rrd_records = [] # public
  58. self._rrd_level = 0
  59. self._rrd_parse_state = 0
  60. self._rrd_chars = ""
  61. self._rrd_capture = False
  62. self._rrd_ds_val = {}
  63. self._rrd_data_row = []
  64. self._rrd_data_row_has_nan = False
  65. def endDocument(self):
  66. pass
  67. # Nasty little ad-hoc state machine to extract the elements that are
  68. # necessary from the 'rrdtool dump' XML output. The same element
  69. # name '<ds>' is used for two different data sets so we need to pay
  70. # some attention to the actual structure to get the ones we want
  71. # and ignore the ones we don't.
  72. def startElement(self, name, attrs):
  73. self._rrd_level = self._rrd_level + 1
  74. self._rrd_capture = False
  75. if self._rrd_level == 1:
  76. if name == "rrd" and self._rrd_parse_state == 0:
  77. self._rrd_parse_state = 1 # In <rrd>
  78. self._rrd_capture = True
  79. self._rrd_chars = ""
  80. elif self._rrd_level == 2:
  81. if self._rrd_parse_state == 1:
  82. if name == "lastupdate":
  83. self._rrd_parse_state = 2 # In <rrd><lastupdate>
  84. self._rrd_capture = True
  85. self._rrd_chars = ""
  86. elif name == "step":
  87. self._rrd_parse_state = 3 # In <rrd><step>
  88. self._rrd_capture = True
  89. self._rrd_chars = ""
  90. elif name == "ds":
  91. self._rrd_parse_state = 4 # In <rrd><ds>
  92. self._rrd_ds_val = {}
  93. self._rrd_chars = ""
  94. elif name == "rra":
  95. self._rrd_parse_state = 5 # In <rrd><rra>
  96. elif self._rrd_level == 3:
  97. if self._rrd_parse_state == 4:
  98. if name == "name":
  99. self._rrd_parse_state = 6 # In <rrd><ds><name>
  100. self._rrd_capture = True
  101. self._rrd_chars = ""
  102. elif name == "type":
  103. self._rrd_parse_state = 7 # In <rrd><ds><type>
  104. self._rrd_capture = True
  105. self._rrd_chars = ""
  106. elif self._rrd_parse_state == 5:
  107. if name == "database":
  108. self._rrd_parse_state = 8 # In <rrd><rra><database>
  109. elif self._rrd_level == 4:
  110. if self._rrd_parse_state == 8:
  111. if name == "row":
  112. self._rrd_parse_state = 9 # In <rrd><rra><database><row>
  113. self._rrd_data_row = []
  114. self._rrd_data_row_has_nan = False
  115. elif self._rrd_level == 5:
  116. if self._rrd_parse_state == 9:
  117. if name == "v":
  118. self._rrd_parse_state = 10 # In <rrd><rra><database><row><v>
  119. self._rrd_capture = True
  120. self._rrd_chars = ""
  121. def endElement(self, name):
  122. self._rrd_capture = False
  123. if self._rrd_parse_state == 10:
  124. self._rrd_capture = self._rrd_level == 6
  125. if self._rrd_level == 5:
  126. if self._rrd_chars == "NaN":
  127. self._rrd_data_row_has_nan = True
  128. else:
  129. self._rrd_data_row.append(self._rrd_chars)
  130. self._rrd_parse_state = 9 # In <rrd><rra><database><row>
  131. elif self._rrd_parse_state == 9:
  132. if self._rrd_level == 4:
  133. if not self._rrd_data_row_has_nan:
  134. self.rrd_records.append(self._rrd_data_row)
  135. self._rrd_parse_state = 8 # In <rrd><rra><database>
  136. elif self._rrd_parse_state == 8:
  137. if self._rrd_level == 3:
  138. self._rrd_parse_state = 5 # In <rrd><rra>
  139. elif self._rrd_parse_state == 7:
  140. if self._rrd_level == 3:
  141. self._rrd_ds_val["type"] = self._rrd_chars
  142. self._rrd_parse_state = 4 # In <rrd><ds>
  143. elif self._rrd_parse_state == 6:
  144. if self._rrd_level == 3:
  145. self._rrd_ds_val["name"] = self._rrd_chars
  146. self._rrd_parse_state = 4 # In <rrd><ds>
  147. elif self._rrd_parse_state == 5:
  148. if self._rrd_level == 2:
  149. self._rrd_parse_state = 1 # In <rrd>
  150. elif self._rrd_parse_state == 4:
  151. if self._rrd_level == 2:
  152. self.rrd_ds.append(self._rrd_ds_val)
  153. self._rrd_parse_state = 1 # In <rrd>
  154. elif self._rrd_parse_state == 3:
  155. if self._rrd_level == 2:
  156. self.rrd_step = long(self._rrd_chars)
  157. self._rrd_parse_state = 1 # In <rrd>
  158. elif self._rrd_parse_state == 2:
  159. if self._rrd_level == 2:
  160. self.rrd_last_update = long(self._rrd_chars)
  161. self._rrd_parse_state = 1 # In <rrd>
  162. elif self._rrd_parse_state == 1:
  163. if self._rrd_level == 1:
  164. self._rrd_parse_state = 0 # At top
  165. if self._rrd_level:
  166. self._rrd_level = self._rrd_level - 1
  167. def characters(self, content):
  168. if self._rrd_capture:
  169. self._rrd_chars = self._rrd_chars + content.strip()
  170. def _make_numeric(value):
  171. try:
  172. value = float(value)
  173. except:
  174. value = ""
  175. return value
  176. def simperf_host_xml_fixup(parser, filter_start_time = None, filter_end_time = None):
  177. # Fixup for GAUGE fields that are really COUNTS. They
  178. # were forced to GAUGE to try to disable rrdtool's
  179. # data interpolation/extrapolation for non-uniform time
  180. # samples.
  181. fixup_tags = [ "cpu_user",
  182. "cpu_nice",
  183. "cpu_sys",
  184. "cpu_idle",
  185. "cpu_waitio",
  186. "cpu_intr",
  187. # "file_active",
  188. # "file_free",
  189. # "inode_active",
  190. # "inode_free",
  191. "netif_in_kb",
  192. "netif_in_pkts",
  193. "netif_in_errs",
  194. "netif_in_drop",
  195. "netif_out_kb",
  196. "netif_out_pkts",
  197. "netif_out_errs",
  198. "netif_out_drop",
  199. "vm_page_in",
  200. "vm_page_out",
  201. "vm_swap_in",
  202. "vm_swap_out",
  203. #"vm_mem_total",
  204. #"vm_mem_used",
  205. #"vm_mem_active",
  206. #"vm_mem_inactive",
  207. #"vm_mem_free",
  208. #"vm_mem_buffer",
  209. #"vm_swap_cache",
  210. #"vm_swap_total",
  211. #"vm_swap_used",
  212. #"vm_swap_free",
  213. "cpu_interrupts",
  214. "cpu_switches",
  215. "cpu_forks" ]
  216. col_count = len(parser.rrd_ds)
  217. row_count = len(parser.rrd_records)
  218. # Process the last row separately, just to make all values numeric.
  219. for j in range(col_count):
  220. parser.rrd_records[row_count - 1][j] = _make_numeric(parser.rrd_records[row_count - 1][j])
  221. # Process all other row/columns.
  222. last_different_row = row_count - 1
  223. current_row = row_count - 2
  224. while current_row >= 0:
  225. # Check for a different value than the previous row. If everything is the same
  226. # then this is probably just a filler/bogus entry.
  227. is_different = False
  228. for j in range(col_count):
  229. parser.rrd_records[current_row][j] = _make_numeric(parser.rrd_records[current_row][j])
  230. if parser.rrd_records[current_row][j] != parser.rrd_records[last_different_row][j]:
  231. # We're good. This is a different row.
  232. is_different = True
  233. if not is_different:
  234. # This is a filler/bogus entry. Just ignore it.
  235. for j in range(col_count):
  236. parser.rrd_records[current_row][j] = float('nan')
  237. else:
  238. # Some tags need to be converted into deltas.
  239. for j in range(col_count):
  240. if parser.rrd_ds[j]["name"] in fixup_tags:
  241. parser.rrd_records[last_different_row][j] = \
  242. parser.rrd_records[last_different_row][j] - parser.rrd_records[current_row][j]
  243. last_different_row = current_row
  244. current_row -= 1
  245. # Set fixup_tags in the first row to 'nan' since they aren't useful anymore.
  246. for j in range(col_count):
  247. if parser.rrd_ds[j]["name"] in fixup_tags:
  248. parser.rrd_records[0][j] = float('nan')
  249. # Add a timestamp to each row and to the catalog. Format and name
  250. # chosen to match other simulator logging (hopefully).
  251. start_time = parser.rrd_last_update - (parser.rrd_step * (row_count - 1))
  252. # Build a filtered list of rrd_records if we are limited to a time range.
  253. filter_records = False
  254. if filter_start_time is not None or filter_end_time is not None:
  255. filter_records = True
  256. filtered_rrd_records = []
  257. if filter_start_time is None:
  258. filter_start_time = start_time * 1000
  259. if filter_end_time is None:
  260. filter_end_time = parser.rrd_last_update * 1000
  261. for i in range(row_count):
  262. record_timestamp = (start_time + (i * parser.rrd_step)) * 1000
  263. parser.rrd_records[i].insert(0, record_timestamp)
  264. if filter_records:
  265. if filter_start_time <= record_timestamp and record_timestamp <= filter_end_time:
  266. filtered_rrd_records.append(parser.rrd_records[i])
  267. if filter_records:
  268. parser.rrd_records = filtered_rrd_records
  269. parser.rrd_ds.insert(0, {"type": "GAUGE", "name": "javascript_timestamp"})
  270. def main(argv=None):
  271. opts, args = getopt.getopt(sys.argv[1:], "i:o:h", ["in=", "out=", "help"])
  272. input_file = sys.stdin
  273. output_file = sys.stdout
  274. for o, a in opts:
  275. if o in ("-i", "--in"):
  276. input_file = open(a, 'r')
  277. if o in ("-o", "--out"):
  278. output_file = open(a, 'w')
  279. if o in ("-h", "--help"):
  280. usage()
  281. sys.exit(0)
  282. # Using the SAX parser as it is at least 4X faster and far, far
  283. # smaller on this dataset than the DOM-based interface in xml.dom.minidom.
  284. # With SAX and a 5.4MB xml file, this requires about seven seconds of
  285. # wall-clock time and 32MB VSZ. With the DOM interface, about 22 seconds
  286. # and over 270MB VSZ.
  287. handler = SimPerfHostXMLParser()
  288. sax.parse(input_file, handler)
  289. if input_file != sys.stdin:
  290. input_file.close()
  291. # Various format fixups: string-to-num, gauge-to-counts, add
  292. # a time stamp, etc.
  293. simperf_host_xml_fixup(handler)
  294. # Create JSONable dict with interesting data and format/print it
  295. print >>output_file, simplejson.dumps({ "step" : handler.rrd_step,
  296. "lastupdate": handler.rrd_last_update * 1000,
  297. "ds" : handler.rrd_ds,
  298. "database" : handler.rrd_records })
  299. return 0
  300. if __name__ == "__main__":
  301. sys.exit(main())