PageRenderTime 36ms CodeModel.GetById 12ms app.highlight 20ms RepoModel.GetById 1ms app.codeStats 0ms

/circuits/web/utils.py

https://bitbucket.org/prologic/circuits/
Python | 386 lines | 280 code | 45 blank | 61 comment | 50 complexity | f1e4ee7c8b207992626e90768971b16e MD5 | raw file
  1# Module:   utils
  2# Date:     13th September 2007
  3# Author:   James Mills, prologic at shortcircuit dot net dot au
  4
  5"""Utilities
  6
  7This module implements utility functions.
  8"""
  9
 10import re
 11import zlib
 12import time
 13import struct
 14from math import sqrt
 15from io import TextIOWrapper
 16from cgi import FieldStorage
 17from collections import MutableMapping
 18
 19try:
 20    from urllib.parse import urljoin as _urljoin
 21except ImportError:
 22    from urlparse import urljoin as _urljoin  # NOQA
 23
 24try:
 25    from urllib.parse import parse_qs as _parse_qs
 26except ImportError:
 27    from cgi import parse_qs as _parse_qs  # NOQA
 28
 29from circuits.six import iterbytes
 30
 31from .exceptions import RangeUnsatisfiable, RequestEntityTooLarge
 32
 33quoted_slash = re.compile("(?i)%2F")
 34image_map_pattern = re.compile("^[0-9]+,[0-9]+$")
 35
 36
 37def average(xs):
 38    return sum(xs) * 1.0 / len(xs)
 39
 40
 41def variance(xs):
 42    avg = average(xs)
 43    return list(map(lambda x: (x - avg)**2, xs))
 44
 45
 46def stddev(xs):
 47    return sqrt(average(variance(xs)))
 48
 49
 50def parse_body(request, response, params):
 51    if "Content-Type" not in request.headers:
 52        request.headers["Content-Type"] = ""
 53
 54    try:
 55        form = FieldStorage(
 56            environ={"REQUEST_METHOD": "POST"},
 57            fp=TextIOWrapper(request.body),
 58            headers=request.headers,
 59            keep_blank_values=True
 60        )
 61    except Exception as e:
 62        if e.__class__.__name__ == 'MaxSizeExceeded':
 63            # Post data is too big
 64            raise RequestEntityTooLarge()
 65        raise
 66
 67    if form.file:
 68        request.body = form.file
 69    else:
 70        params.update(dictform(form))
 71
 72
 73def parse_qs(query_string, keep_blank_values=True):
 74    """parse_qs(query_string) -> dict
 75
 76    Build a params dictionary from a query_string.
 77    If keep_blank_values is True (the default), keep
 78    values that are blank.
 79    """
 80
 81    if image_map_pattern.match(query_string):
 82        # Server-side image map. Map the coords to "x" and "y"
 83        # (like CGI::Request does).
 84        pm = query_string.split(",")
 85        return {"x": int(pm[0]), "y": int(pm[1])}
 86    else:
 87        pm = _parse_qs(query_string, keep_blank_values)
 88        return dict((k, v[0]) for k, v in pm.items() if v)
 89
 90
 91def dictform(form):
 92    d = {}
 93    for key in list(form.keys()):
 94        values = form[key]
 95        if isinstance(values, list):
 96            d[key] = []
 97            for item in values:
 98                if item.filename is not None:
 99                    value = item  # It's a file upload
100                else:
101                    value = item.value  # It's a regular field
102                d[key].append(value)
103        else:
104            if values.filename is not None:
105                value = values  # It's a file upload
106            else:
107                value = values.value  # It's a regular field
108            d[key] = value
109    return d
110
111
112def compress(body, compress_level):
113    """Compress 'body' at the given compress_level."""
114
115    # Header
116    yield b"\037\213\010\0" \
117        + struct.pack("<L", int(time.time())) \
118        + b"\002\377"
119
120    size = 0
121    crc = zlib.crc32(b"")
122
123    zobj = zlib.compressobj(
124        compress_level,
125        zlib.DEFLATED,
126        -zlib.MAX_WBITS,
127        zlib.DEF_MEM_LEVEL,
128        0,
129    )
130
131    for chunk in body:
132        if not isinstance(chunk, bytes):
133            chunk = chunk.encode("utf-8")
134
135        size += len(chunk)
136        crc = zlib.crc32(chunk, crc)
137        yield zobj.compress(chunk)
138
139    yield zobj.flush() \
140        + struct.pack("<l", crc) \
141        + struct.pack("<L", size & 0xFFFFFFFF)
142
143
144def get_ranges(headervalue, content_length):
145    """Return a list of (start, stop) indices from a Range header, or None.
146
147    Each (start, stop) tuple will be composed of two ints, which are suitable
148    for use in a slicing operation. That is, the header "Range: bytes=3-6",
149    if applied against a Python string, is requesting resource[3:7]. This
150    function will return the list [(3, 7)].
151
152    If this function returns an empty list, you should return HTTP 416.
153    """
154
155    if not headervalue:
156        return None
157
158    result = []
159    bytesunit, byteranges = headervalue.split("=", 1)
160    for brange in byteranges.split(","):
161        start, stop = [x.strip() for x in brange.split("-", 1)]
162        if start:
163            if not stop:
164                stop = content_length - 1
165            start, stop = list(map(int, (start, stop)))
166            if start >= content_length:
167                # From rfc 2616 sec 14.16:
168                # "If the server receives a request (other than one
169                # including an If-Range request-header field) with an
170                # unsatisfiable Range request-header field (that is,
171                # all of whose byte-range-spec values have a first-byte-pos
172                # value greater than the current length of the selected
173                # resource), it SHOULD return a response code of 416
174                # (Requested range not satisfiable)."
175                continue
176            if stop < start:
177                # From rfc 2616 sec 14.16:
178                # "If the server ignores a byte-range-spec because it
179                # is syntactically invalid, the server SHOULD treat
180                # the request as if the invalid Range header field
181                # did not exist. (Normally, this means return a 200
182                # response containing the full entity)."
183                return None
184            # Prevent duplicate ranges. See Issue #59
185            if (start, stop + 1) not in result:
186                result.append((start, stop + 1))
187        else:
188            if not stop:
189                # See rfc quote above.
190                return None
191            # Negative subscript (last N bytes)
192            # Prevent duplicate ranges. See Issue #59
193            if (content_length - int(stop), content_length) not in result:
194                result.append((content_length - int(stop), content_length))
195
196    # Can we satisfy the requested Range?
197    # If we have an exceedingly high standard deviation
198    # of Range(s) we reject the request.
199    # See Issue #59
200
201    if len(result) > 1 and stddev([x[1] - x[0] for x in result]) > 2.0:
202        raise RangeUnsatisfiable()
203
204    return result
205
206
207class IOrderedDict(dict, MutableMapping):
208    'Dictionary that remembers insertion order with insensitive key'
209    # An inherited dict maps keys to values.
210    # The inherited dict provides __getitem__, __len__, __contains__, and get.
211    # The remaining methods are order-aware.
212    # Big-O running times for all methods are the same as for regular dictionaries.
213
214    # The internal self.__map dictionary maps keys to links in a doubly linked list.
215    # The circular doubly linked list starts and ends with a sentinel element.
216    # The sentinel element never gets deleted (this simplifies the algorithm).
217    # Each link is stored as a list of length three:  [PREV, NEXT, KEY].
218
219    def __init__(self, *args, **kwds):
220        '''Initialize an ordered dictionary.  Signature is the same as for
221        regular dictionaries, but keyword arguments are not recommended
222        because their insertion order is arbitrary.
223
224        '''
225        if len(args) > 1:
226            raise TypeError('expected at most 1 arguments, got %d' % len(args))
227        try:
228            self.__root
229        except AttributeError:
230            self.__root = root = [None, None, None]     # sentinel node
231            PREV = 0
232            NEXT = 1
233            root[PREV] = root[NEXT] = root
234            self.__map = {}
235        self.__lower = {}
236        self.update(*args, **kwds)
237
238    def __setitem__(self, key, value, PREV=0, NEXT=1, dict_setitem=dict.__setitem__):
239        'od.__setitem__(i, y) <==> od[i]=y'
240        # Setting a new item creates a new link which goes at the end of the linked
241        # list, and the inherited dictionary is updated with the new key/value pair.
242        if key not in self:
243            root = self.__root
244            last = root[PREV]
245            last[NEXT] = root[PREV] = self.__map[key] = [last, root, key]
246            self.__lower[key.lower()] = key
247        key = self.__lower[key.lower()]
248        dict_setitem(self, key, value)
249
250    def __delitem__(self, key, PREV=0, NEXT=1, dict_delitem=dict.__delitem__):
251        'od.__delitem__(y) <==> del od[y]'
252        # Deleting an existing item uses self.__map to find the link which is
253        # then removed by updating the links in the predecessor and successor nodes.
254        if key in self:
255            key = self.__lower.pop(key.lower())
256
257        dict_delitem(self, key)
258        link = self.__map.pop(key)
259        link_prev = link[PREV]
260        link_next = link[NEXT]
261        link_prev[NEXT] = link_next
262        link_next[PREV] = link_prev
263
264    def __getitem__(self, key, dict_getitem=dict.__getitem__):
265        if key in self:
266            key = self.__lower.get(key.lower())
267        return dict_getitem(self, key)
268
269    def __contains__(self, key):
270        return key.lower() in self.__lower
271
272    def __iter__(self, NEXT=1, KEY=2):
273        'od.__iter__() <==> iter(od)'
274        # Traverse the linked list in order.
275        root = self.__root
276        curr = root[NEXT]
277        while curr is not root:
278            yield curr[KEY]
279            curr = curr[NEXT]
280
281    def __reversed__(self, PREV=0, KEY=2):
282        'od.__reversed__() <==> reversed(od)'
283        # Traverse the linked list in reverse order.
284        root = self.__root
285        curr = root[PREV]
286        while curr is not root:
287            yield curr[KEY]
288            curr = curr[PREV]
289
290    def __reduce__(self):
291        'Return state information for pickling'
292        items = [[k, self[k]] for k in self]
293        tmp = self.__map, self.__root
294        del self.__map, self.__root
295        inst_dict = vars(self).copy()
296        self.__map, self.__root = tmp
297        if inst_dict:
298            return (self.__class__, (items,), inst_dict)
299        return self.__class__, (items,)
300
301    def clear(self):
302        'od.clear() -> None.  Remove all items from od.'
303        try:
304            for node in self.__map.values():
305                del node[:]
306            self.__root[:] = [self.__root, self.__root, None]
307            self.__map.clear()
308        except AttributeError:
309            pass
310        dict.clear(self)
311
312    def get(self, key, default=None):
313        if key in self:
314            return self[key]
315        return default
316
317    setdefault = MutableMapping.setdefault
318    update = MutableMapping.update
319    pop = MutableMapping.pop
320    keys = MutableMapping.keys
321    values = MutableMapping.values
322    items = MutableMapping.items
323    __ne__ = MutableMapping.__ne__
324
325    def popitem(self, last=True):
326        '''od.popitem() -> (k, v), return and remove a (key, value) pair.
327        Pairs are returned in LIFO order if last is true or FIFO order if false.
328
329        '''
330        if not self:
331            raise KeyError('dictionary is empty')
332        key = next(reversed(self) if last else iter(self))
333        value = self.pop(key)
334        return key, value
335
336    def __repr__(self):
337        'od.__repr__() <==> repr(od)'
338        if not self:
339            return '%s()' % (self.__class__.__name__,)
340        return '%s(%r)' % (self.__class__.__name__, list(self.items()))
341
342    def copy(self):
343        'od.copy() -> a shallow copy of od'
344        return self.__class__(self)
345
346    @classmethod
347    def fromkeys(cls, iterable, value=None):
348        '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S
349        and values equal to v (which defaults to None).
350
351        '''
352        d = cls()
353        for key in iterable:
354            d[key] = value
355        return d
356
357    def __eq__(self, other):
358        '''od.__eq__(y) <==> od==y.  Comparison to another OD is order-sensitive
359        while comparison to a regular mapping is order-insensitive.
360
361        '''
362        if isinstance(other, IOrderedDict):
363            return (
364                len(self) == len(other)
365                and set(self.items()) == set(other.items())
366            )
367        return dict.__eq__(self, other)
368
369    def __del__(self):
370        self.clear()                # eliminate cyclical references
371
372
373def is_ssl_handshake(buf):
374    """Detect an SSLv2 or SSLv3 handshake"""
375
376    # SSLv3
377
378    v = buf[:3]
379    if v in ["\x16\x03\x00", "\x16\x03\x01", "\x16\x03\x02"]:
380        return True
381
382    # SSLv2
383
384    v = list(iterbytes(buf))[:2]
385    if (v[0] & 0x80 == 0x80) and ((v[0] & 0x7f) << 8 | v[1]) > 9:
386        return True