PageRenderTime 83ms CodeModel.GetById 40ms app.highlight 2ms RepoModel.GetById 39ms app.codeStats 0ms

/mysql_watcher/mysql_logger

https://bitbucket.org/lindenlab/apiary/
Python | 156 lines | 97 code | 23 blank | 36 comment | 15 complexity | 35aaac83b4c8e955480ea378126e3aa2 MD5 | raw file
  1#!/usr/bin/env python
  2#
  3# $LicenseInfo:firstyear=2010&license=mit$
  4# 
  5# Copyright (c) 2010, Linden Research, Inc.
  6# 
  7# Permission is hereby granted, free of charge, to any person obtaining a copy
  8# of this software and associated documentation files (the "Software"), to deal
  9# in the Software without restriction, including without limitation the rights
 10# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 11# copies of the Software, and to permit persons to whom the Software is
 12# furnished to do so, subject to the following conditions:
 13# 
 14# The above copyright notice and this permission notice shall be included in
 15# all copies or substantial portions of the Software.
 16# 
 17# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 18# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 19# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 20# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 21# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 22# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 23# THE SOFTWARE.
 24# $/LicenseInfo$
 25#
 26
 27"""
 28Log all queries hitting a particular mysql database
 29"""
 30
 31try:
 32    import psyco
 33    psyco.full()
 34except:
 35    pass
 36
 37import curses
 38import curses.wrapper
 39import getopt
 40import os.path
 41import re
 42import socket
 43import sys
 44import time
 45
 46from dblibs.dbutil import LLQueryStream, remote_mysql_stream
 47
 48LOG_ROTATION_INTERVAL=3600
 49MAX_LOGS = 36
 50
 51QUERY_LOG_RE = re.compile("query.log.(.+).gz")
 52def rotate_logs(log_path, query_log_file):
 53    # Fork to do the actual rotation/compression
 54    print "Rotating query logs"
 55    if query_log_file:
 56        query_log_file.close()
 57    need_gzip = False
 58
 59    if os.path.exists(log_path+"/query.log"):
 60        os.rename(log_path+"/query.log", log_path+"/query.log.tmp")
 61        need_gzip = True
 62    
 63    query_log_file = open("%s/query.log" % log_path, "w")
 64
 65    pid = os.fork()
 66    if pid:
 67        return query_log_file
 68
 69    # Child process actually does the log rotation
 70    # Delete the oldest
 71    log_filename = log_path+"/query.log.%d.gz" % (MAX_LOGS)
 72    if os.path.exists(log_filename):
 73        os.remove(log_filename)
 74
 75    for i in range(0, MAX_LOGS):
 76        # Count down from the max and rename
 77        n = MAX_LOGS - i
 78        log_filename = log_path+"/query.log.%d.gz" % n
 79        if os.path.exists(log_filename):
 80            os.rename(log_path + ("/query.log.%d.gz" % n), log_path + ("/query.log.%d.gz" % (n+1)))
 81
 82    if need_gzip:
 83        # Compress the "first" log (query.log.tmp)
 84        os.rename(log_path + "/query.log.tmp", log_path + "/query.log.1")
 85        os.system('gzip -f %s' % (log_path + "/query.log.1"))
 86    print "Done rotating logs!"
 87    sys.exit(0)
 88
 89
 90def watch_host(query_stream, host):
 91    "Watches query traffic for a particular host.  Returns the overall query counts when exited by breaking"
 92
 93    # Make output path
 94    output_path = "./%s" % host
 95    os.system("mkdir -p %s" % output_path)
 96    query_log_file = rotate_logs(output_path, None)
 97
 98    last_log_time = time.time()
 99
100    done = False
101    count = 0
102    try:
103        while not done:
104            (event_type, query) = query_stream.getNextEvent()
105
106            # Use the start time to determine which hour bin to put the query into
107            start_time = query.mStartTime
108            start_hour = time.localtime(start_time)[3]
109            
110            if event_type == "QueryStart":
111                query_log_file.write("%f\t%s:%d\t%s\tQueryStart\n" % (query.mStartTime, query.mData['host'], query.mData['port'], query.mData['host_clean']))
112                query_log_file.write("%s\n" % (query.mData['query']))
113                query_log_file.write("**************************************\n")
114                count += 1
115            elif (event_type == "QueryResponse"):
116                query_log_file.write("%f\t%s:%d\t%s\tQueryResponse\n" % (query.mResponseTime, query.mData['host'], query.mData['port'], query.mData['host_clean']))
117                query_log_file.write("%s\n" % (query.mData['query']))
118                query_log_file.write("**************************************\n")
119            elif event_type == "Quit":
120                # Quit is an "instantaneous" query, both start and response
121                query_log_file.write("%f\t%s:%d\t%s\tQuit\n" % (query.mStartTime, query.mData['host'], query.mData['port'], query.mData['host_clean']))
122                query_log_file.write("%s\n" % (query.mData['query']))
123                query_log_file.write("**************************************\n")
124                continue
125            if not (count % 1000):
126                try:
127                    os.waitpid(-1, os.WNOHANG)
128                except OSError:
129                    pass
130                if (time.time() - last_log_time) > LOG_ROTATION_INTERVAL:
131                    last_log_time = time.time()
132                    query_log_file = rotate_logs(output_path, query_log_file)
133
134            
135    except KeyboardInterrupt:
136        pass
137    query_log_file.close()
138
139
140if __name__ == "__main__":
141    opts, args = getopt.getopt(sys.argv[1:], "", ["host="])
142
143    host = None
144    for o, a in opts:
145        if o in ("--host"):
146            host = a
147    if not host:
148        print "Specify a host using --host="
149        sys.exit(1)
150
151    # Start up the stream from the target host and create a file
152    # that we can hand to LLQueryStream
153    query_stream_file = remote_mysql_stream(host)
154    query_stream = LLQueryStream(query_stream_file)
155
156    watch_host(query_stream, host)