PageRenderTime 235ms CodeModel.GetById 81ms app.highlight 52ms RepoModel.GetById 99ms app.codeStats 0ms

/historical/sqllog.py

https://bitbucket.org/lindenlab/apiary/
Python | 344 lines | 308 code | 11 blank | 25 comment | 11 complexity | db9152e15900ba1b4652261f4205f357 MD5 | raw file
  1#
  2# $LicenseInfo:firstyear=2010&license=mit$
  3# 
  4# Copyright (c) 2010, Linden Research, Inc.
  5# 
  6# Permission is hereby granted, free of charge, to any person obtaining a copy
  7# of this software and associated documentation files (the "Software"), to deal
  8# in the Software without restriction, including without limitation the rights
  9# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 10# copies of the Software, and to permit persons to whom the Software is
 11# furnished to do so, subject to the following conditions:
 12# 
 13# The above copyright notice and this permission notice shall be included in
 14# all copies or substantial portions of the Software.
 15# 
 16# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 17# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 18# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 19# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 20# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 21# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 22# THE SOFTWARE.
 23# $/LicenseInfo$
 24#
 25
 26import heapq
 27import re
 28import sys
 29
 30from mergetools import imerge
 31from timestamp import TimeStamp
 32
 33__all__ = [
 34        'Event',
 35        'CoalescedEvent',
 36        'Sequence',
 37        'parse_stanza',
 38        'EventReader',
 39        'input_events',
 40        'FollowSequences',
 41        'CoalesceSequences'
 42    ]
 43
 44headerRE = re.compile(r'^(\d+\.\d+)\t([\d.:]+)\t(\S+)\t(\w+)$')
 45breakRE = re.compile(r'^\*{3,}$')
 46
 47class Event(object):
 48    # state values
 49    Query = 'QueryStart'
 50    Response = 'QueryResponse'
 51    End = 'Quit'
 52    
 53    def __init__(self, time, id, source, state, body):
 54        self.time = time
 55        if type(time) is not TimeStamp:
 56            self.time = TimeStamp(time)
 57        self.id = id
 58        self.source = source
 59        self.state = state
 60        self.body = body
 61    
 62    def __cmp__(self, other):
 63        c = cmp(self.time, other.time)
 64        if c == 0:
 65            if self.state == "Quit" and other.state != "Quit":
 66                return 1
 67            if self.state != "Quit" and other.state == "Quit":
 68                return -1
 69        return c
 70        
 71    def __str__(self):
 72        return ("%s\t%s\t%s\t%s\n%s\n**************************************\n"
 73            % (self.time, self.id, self.source, self.state, self.body))
 74    
 75    def events(self):
 76        es = []
 77        for event in self.body.split("\n+++\n"):
 78            parts = event.split(":", 1)
 79            if len(parts) == 2:
 80                (time, body) = parts
 81                status = Event.Query
 82                if body == "Quit":
 83                    status = Event.End
 84                es.append(Event(time, self.id, self.source, status, body))
 85        return es
 86
 87class CoalescedEvent(Event):
 88    Sequence = 'Sequence'
 89    
 90    def __init__(self, shelf_life=None, max_life=None):
 91        self.time = None
 92        self.id = None
 93        self.source = None
 94        self.state = CoalescedEvent.Sequence
 95        self.body = ""
 96        self.ended = False
 97        self.lasttime = None
 98        self.staletime = None
 99        self.maxtime = None
100        self.shelf_life = None
101        if shelf_life:
102            self.shelf_life = TimeStamp(shelf_life)
103        self.max_life = None
104        if max_life:
105            self.max_life = TimeStamp(max_life)
106    
107    def add(self, event):
108        if self.time is None:
109            self.time = event.time
110            if self.max_life:
111                self.maxtime = self.time + self.max_life
112            self.id = event.id
113            self.source = event.source
114        self.lasttime = event.time
115        if self.shelf_life:
116            self.staletime = self.lasttime + self.shelf_life
117        if not event.state == Event.End:
118            self.body += "%s:%s\n+++\n" % (event.time, event.body) 
119        else:
120            self.body += "%s:Quit\n+++\n" % (event.time)
121            self.ended = True
122    
123    def endIfNeeded(self):
124        if not self.ended:
125            self.add(Event(self.lasttime, self.id, self.source,
126                            Event.End, 'Quit'))
127
128
129def parse_stanza(input):
130    match = None
131    while not match:
132        line = input.readline()
133        if line == '':  # empty string means EOF
134            return None
135        match = headerRE.match(line)
136    (time, id, source, state) = match.groups()
137    
138    body = ''
139    while True:
140        line = input.readline()
141        if line == '':
142            break
143        if breakRE.match(line):
144            break
145        body += line
146    
147    return Event(float(time), id, source, state, body)
148    
149class Sequence(object):
150    def __init__(self):
151        self._count = 0
152        self._time_start = None
153        self._last_event = None
154    
155    def note(self, event):
156        self._count += 1
157        if self._time_start is None:
158            self._time_start = event.time
159        self._last_event = event
160    
161    def count(self):
162        return self._count
163    
164    def time(self):
165        return self._last_event.time - self._time_start
166    
167    def timeto(self, event):
168        if self._last_event is None:
169            return None
170        return event.time - self._last_event.time
171        
172    def ended(self):
173        return self._last_event.state == Event.End
174        
175    def generateEnd(self, t=None):
176        e = self._last_event
177        if t is None:
178            t = e.time
179        return Event(t, e.id, e.source, Event.End, "")
180
181
182class EventReader(object):
183    def __init__(self, input):
184        self._input = input
185    
186    def __iter__(self):
187        while True:
188            s = parse_stanza(self._input)
189            if s is None:
190                return
191            if s.state == CoalescedEvent.Sequence:
192                for t in s.events():
193                    yield t
194            else:
195                yield s
196
197def input_spec_to_file(spec):
198    if spec == '-':
199        return sys.stdin
200    return file(spec)
201
202def input_events(specs):
203    if len(specs) == 0:
204        return iter(EventReader(sys.stdin))
205    evs = map(EventReader, map(input_spec_to_file, specs))
206    return imerge(*evs)
207
208
209class FollowSequences(object):
210    def replay(self, events):
211        connections = { }
212        lastt = None;
213        for e in events:
214            id = e.id
215            lastt = e.time
216            if id not in connections:
217                s = connections[id] = Sequence()
218                self.addingSequence(s, e)
219            else:
220                s = connections[id]
221            self.notingEvent(s, e)
222            s.note(e)
223            if s.ended():
224                self.removingSequence(s, e)
225                del connections[id]
226            if False:
227                expired = []
228                for (id,s) in connections.iteritems():
229                    w = s.timeto(e)
230                    if w and float(w) > 60.0:
231                        expired.append((id,s))
232                for (id,s) in expired:
233                    f = s.generateEnd(e.time)
234                    self.forcedEnd(s, f)
235                    self.removingSequence(s, f)
236                    del connections[id]
237        for s in connections.itervalues():
238            f = s.generateEnd(lastt)
239            self.forcedEnd(s, f)
240            self.removingSequence(s, f)
241            
242    def addingSequence(self, s, e):
243        pass
244    
245    def notingEvent(self, s, e):
246        pass
247    
248    def forcedEnd(self, s, e):
249        pass
250    
251    def removingSequence(self, s, e):
252        pass
253
254
255class CoalesceSequences(object):
256    def __init__(self):
257        self.connections = { }
258        self.bytime = [ ]
259        self.starttime = None
260        self.lasttime = None
261        
262    def heartbeat(self, n):
263        sys.stderr.write("%s: %d events... (%d connections, %d waiting)\n"
264            % (str(self.lasttime - self.starttime), n, len(self.connections), len(self.bytime)))
265        n = 0
266        i = 0
267        l = len(self.bytime)
268        s = ""
269        while n < 5 and i < l:
270            en = 0
271            while i < l and self.bytime[i].ended:
272                en += 1
273                i += 1
274            if en > 0:
275                s += " : --%d--" % en
276            else:
277                n += 1
278                s += " : %s(%s)" % (self.bytime[i].id, str(self.lasttime - self.bytime[i].lasttime))
279                i += 1
280        sys.stderr.write("                          ")
281        sys.stderr.write(s)
282        sys.stderr.write("\n")
283
284    def age_out(self, c):
285        if c.staletime and self.lasttime >= c.staletime:
286            sys.stderr.write("                           expiring %s, stale\n" % c.id)
287        elif c.maxtime and self.lasttime >= c.maxtime:
288            sys.stderr.write("                           expiring %s, maxed out\n" % c.id)
289        else:
290            return False
291        c.endIfNeeded()
292        del self.connections[c.id]
293        return True
294        
295    def flush_completed(self):
296        bytime = self.bytime
297        
298        while bytime:
299            c = bytime[0]
300            if not c.ended:
301                if not self.age_out(c):
302                    return
303            heapq.heappop(bytime)
304            self.fullSequence(c)
305#            sys.stderr.write("                           done %s\n" % c.id)
306
307
308    def replay(self, events):
309        n = 0;
310        connections = self.connections
311        bytime = self.bytime
312        for e in events:
313            id = e.id
314            self.lasttime = e.time
315            if self.starttime is None:
316                self.starttime = self.lasttime
317
318            n += 1
319            if n % 10000 == 0:
320                self.heartbeat(n)
321            
322            if id in connections:
323                c = connections[id]
324                self.age_out(c)
325
326            if id not in connections:
327                c = connections[id] = CoalescedEvent(30.0, 180.0)
328                c.add(e)
329                heapq.heappush(bytime, c)
330            else:
331                c.add(e)
332
333            if e.state == Event.End:
334                del connections[id]
335
336            self.flush_completed()
337
338        for d in connections.itervalues():
339            d.endIfNeeded()
340        self.flush_completed()
341                    
342    def fullSequence(self, e):
343        pass
344