PageRenderTime 184ms CodeModel.GetById 80ms app.highlight 2ms RepoModel.GetById 99ms app.codeStats 0ms

/mysql_watcher/log_watcher

https://bitbucket.org/lindenlab/apiary/
Python | 186 lines | 120 code | 23 blank | 43 comment | 15 complexity | f42556873849fab90118eb17bc0e9d77 MD5 | raw file
  1#!/usr/bin/env python
  2#
  3# $LicenseInfo:firstyear=2010&license=mit$
  4# 
  5# Copyright (c) 2010, Linden Research, Inc.
  6# 
  7# Permission is hereby granted, free of charge, to any person obtaining a copy
  8# of this software and associated documentation files (the "Software"), to deal
  9# in the Software without restriction, including without limitation the rights
 10# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 11# copies of the Software, and to permit persons to whom the Software is
 12# furnished to do so, subject to the following conditions:
 13# 
 14# The above copyright notice and this permission notice shall be included in
 15# all copies or substantial portions of the Software.
 16# 
 17# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 18# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 19# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 20# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 21# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 22# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 23# THE SOFTWARE.
 24# $/LicenseInfo$
 25#
 26
 27"""
 28Summarizes data collected by mysql_logger
 29"""
 30
 31try:
 32    import psyco
 33    psyco.full()
 34except:
 35    pass
 36
 37import curses
 38import curses.wrapper
 39import getopt
 40import os
 41import re
 42import socket
 43import sys
 44import time
 45import traceback
 46
 47from llbase import llsd
 48from dblibs.dbutil import LLQueryStream, LLQueryStatMap, LLBinnedQueryStats, LLLogQueryStream, LLLogIter
 49from dblibs.dbbrowser import *
 50
 51DISPLAY_INTERVAL = 5.0
 52DUMP_INTERVAL = 360.0
 53
 54
 55def watch_host(stdscr, host, query_metadata):
 56    "Watches query traffic for a particular host.  Returns the overall query counts when exited by breaking"
 57
 58    #
 59    # Options/parameters
 60    #
 61    output_path = "./" + host
 62    # Generate a chronologically sorted list of all log files in the directory.
 63    MAX_LOGS=36
 64    log_files = []
 65    for i in range(0, MAX_LOGS):
 66        # Count down from the max and rename
 67        n = MAX_LOGS - i
 68        
 69        log_filename = output_path + "/query.log.%d.gz" % n
 70        if os.path.exists(log_filename):
 71            log_files.append(log_filename)
 72    
 73    log_filename = output_path + "/query.log"
 74    if os.path.exists(log_filename):
 75        log_files.append(log_filename)
 76
 77    # Start listening to data stream
 78    query_stream = LLLogQueryStream(LLLogIter(log_files))
 79
 80    # Curses configuration stuff
 81    size = stdscr.getmaxyx()
 82    stdscr.nodelay(True) # Non-blocking keyboard input
 83
 84    sort_by = "total_time"
 85
 86    count = 0
 87    cur_time = 0
 88    last_display_time = time.time()
 89    last_dump_time = time.time()
 90    last_count = 0
 91    total_stats = LLQueryStatMap("Total", time.time())
 92    binned_stats = LLBinnedQueryStats()
 93    display_stats = total_stats
 94
 95    query_browser = LLQueryBrowser(stdscr, total_stats, query_metadata)
 96
 97
 98    done = False
 99    try:
100        while not done:
101            (event_type, query) = query_stream.getNextEvent()
102
103            if event_type == "QueryStart":
104                total_stats.queryStart(query)
105                binned_stats.queryStart(query)
106            elif (event_type == "QueryResponse"):
107                total_stats.queryResponse(query)
108                binned_stats.queryResponse(query)
109                count += 1
110            elif event_type == "Quit":
111                # Quit is an "instantaneous" query, both start and response
112                total_stats.queryStart(query)
113                total_stats.queryResponse(query)
114                binned_stats.queryStart(query)
115                binned_stats.queryResponse(query)
116                continue
117            elif event_type == None:
118                # no more data
119                # Open the next file
120                if len(log_files):
121                    query_stream 
122                time.sleep(0.01)
123
124            # Spam
125            if (0 == (count % 100)) or (None == event_type):
126                # Check keyboard input
127                c = 0
128                done = False
129                while -1 != c:
130                    c = stdscr.getch()
131                    if -1 == c:
132                        pass
133                    elif query_browser.handleKey(c):
134                        query_browser.redraw()
135                        pass
136                    elif c == ord('q'):
137                        # Quit
138                        done = True
139                        break
140                    elif c == ord(' '):
141                        # Switch which bins we're displaying
142                        query_browser.setQueryMap(display_stats)
143                        query_browser.redraw()
144                    elif c == ord('d'):
145                        # Dump output
146                        total_stats.dumpTiming("%s/query_timing.txt" % output_path)
147                        total_stats.dumpLLSD("%s/query_dump.llsd" % output_path)
148                        binned_stats.dumpTiming(output_path)
149                        binned_stats.dumpLLSD(output_path)
150                    else:
151                        print "Pressed key %s" % c
152
153                cur_time = time.time()
154                if (cur_time - last_display_time > DISPLAY_INTERVAL):
155                    last_display_time = cur_time
156                    last_count = count
157                    query_browser.setQueryMap(display_stats)
158                    query_browser.redraw()
159                if (cur_time - last_dump_time > DUMP_INTERVAL):
160                    last_dump_time = cur_time
161                    total_stats.dumpTiming("%s/query_timing.txt" % output_path)
162                    total_stats.dumpLLSD("%s/query_dump.llsd" % output_path)
163                    binned_stats.dumpTiming(output_path)
164                    binned_stats.dumpLLSD(output_path)
165    except:
166        traceback.print_exc()
167
168    total_stats.dumpTiming("%s/query_timing.txt" % output_path)
169    total_stats.dumpLLSD("%s/query_dump.llsd" % output_path)
170
171
172if __name__ == "__main__":
173    opts, args = getopt.getopt(sys.argv[1:], "", ["host="])
174
175    host = None
176    for o, a in opts:
177        if o in ("--host"):
178            host = a
179
180    if not host:
181        raise "No host specified"
182
183    query_metadata = LLQueryMetadata("./query_info.llsd")
184
185    #watch_host(None, host, query_metadata)
186    curses.wrapper(watch_host, host, query_metadata)