/weboob/tools/misc.py

https://gitlab.com/phyks/weboob · Python · 185 lines · 154 code · 9 blank · 22 comment · 3 complexity · c36de2f30956757389c703400bc393f2 MD5 · raw file

  1. # -*- coding: utf-8 -*-
  2. # Copyright(C) 2010-2014 Romain Bignon
  3. #
  4. # This file is part of weboob.
  5. #
  6. # weboob is free software: you can redistribute it and/or modify
  7. # it under the terms of the GNU Affero General Public License as published by
  8. # the Free Software Foundation, either version 3 of the License, or
  9. # (at your option) any later version.
  10. #
  11. # weboob is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # GNU Affero General Public License for more details.
  15. #
  16. # You should have received a copy of the GNU Affero General Public License
  17. # along with weboob. If not, see <http://www.gnu.org/licenses/>.
  18. from time import time, sleep
  19. import locale
  20. import os
  21. import sys
  22. import traceback
  23. import types
  24. # keep compatibility
  25. from .compat import unicode
  26. __all__ = ['get_backtrace', 'get_bytes_size', 'iter_fields',
  27. 'to_unicode', 'limit', 'find_exe']
  28. def get_backtrace(empty="Empty backtrace."):
  29. """
  30. Try to get backtrace as string.
  31. Returns "Error while trying to get backtrace" on failure.
  32. """
  33. try:
  34. info = sys.exc_info()
  35. trace = traceback.format_exception(*info)
  36. sys.exc_clear()
  37. if trace[0] != "None\n":
  38. return "".join(trace)
  39. except:
  40. return "Error while trying to get backtrace"
  41. return empty
  42. def get_bytes_size(size, unit_name):
  43. r"""Converts a unit and a number into a number of bytes.
  44. >>> get_bytes_size(2, 'KB')
  45. 2048.0
  46. """
  47. unit_data = {
  48. 'bytes': 1,
  49. 'KB': 1024,
  50. 'KiB': 1024,
  51. 'MB': 1024 * 1024,
  52. 'MiB': 1024 * 1024,
  53. 'GB': 1024 * 1024 * 1024,
  54. 'GiB': 1024 * 1024 * 1024,
  55. 'TB': 1024 * 1024 * 1024 * 1024,
  56. 'TiB': 1024 * 1024 * 1024 * 1024,
  57. }
  58. return float(size * unit_data.get(unit_name, 1))
  59. def iter_fields(obj):
  60. for attribute_name in dir(obj):
  61. if attribute_name.startswith('_'):
  62. continue
  63. attribute = getattr(obj, attribute_name)
  64. if not isinstance(attribute, types.MethodType):
  65. yield attribute_name, attribute
  66. def to_unicode(text):
  67. r"""
  68. >>> to_unicode('ascii')
  69. u'ascii'
  70. >>> to_unicode(u'utf\xe9'.encode('UTF-8'))
  71. u'utf\xe9'
  72. >>> to_unicode(u'unicode')
  73. u'unicode'
  74. """
  75. if isinstance(text, unicode):
  76. return text
  77. if not isinstance(text, str):
  78. try:
  79. text = str(text)
  80. except UnicodeError:
  81. return unicode(text)
  82. try:
  83. return unicode(text, 'utf-8')
  84. except UnicodeError:
  85. try:
  86. return unicode(text, 'iso-8859-15')
  87. except UnicodeError:
  88. return unicode(text, 'windows-1252', 'replace')
  89. def guess_encoding(stdio):
  90. try:
  91. encoding = stdio.encoding or locale.getpreferredencoding()
  92. except AttributeError:
  93. encoding = None
  94. # ASCII or ANSII is most likely a user mistake
  95. if not encoding or encoding.lower() == 'ascii' or encoding.lower().startswith('ansi'):
  96. encoding = 'UTF-8'
  97. return encoding
  98. def limit(iterator, lim):
  99. """Iterate on the lim first elements of iterator."""
  100. count = 0
  101. iterator = iter(iterator)
  102. while count < lim:
  103. yield iterator.next()
  104. count += 1
  105. def ratelimit(group, delay):
  106. """
  107. Simple rate limiting.
  108. Waits if the last call of lastlimit with this group name was less than
  109. delay seconds ago. The rate limiting is global, shared between any instance
  110. of the application and any call to this function sharing the same group
  111. name. The same group name should not be used with different delays.
  112. This function is intended to be called just before the code that should be
  113. rate-limited.
  114. This function is not thread-safe. For reasonably non-critical rate
  115. limiting (like accessing a website), it should be sufficient nevertheless.
  116. @param group [string] rate limiting group name, alphanumeric
  117. @param delay [int] delay in seconds between each call
  118. """
  119. from tempfile import gettempdir
  120. path = os.path.join(gettempdir(), 'weboob_ratelimit.%s' % group)
  121. while True:
  122. try:
  123. offset = time() - os.stat(path).st_mtime
  124. except OSError:
  125. with open(path, 'w'):
  126. pass
  127. offset = 0
  128. if delay < offset:
  129. break
  130. sleep(delay - offset)
  131. os.utime(path, None)
  132. def find_exe(basename):
  133. """
  134. Find the path to an executable by its base name (such as 'gpg').
  135. The executable can be overriden using an environment variable in the form
  136. `NAME_EXECUTABLE` where NAME is the specified base name in upper case.
  137. If the environment variable is not provided, the PATH will be searched
  138. both without and with a ".exe" suffix for Windows compatibility.
  139. If the executable can not be found, None is returned.
  140. """
  141. env_exe = os.getenv('%s_EXECUTABLE' % basename.upper())
  142. if env_exe and os.path.exists(env_exe) and os.access(env_exe, os.X_OK):
  143. return env_exe
  144. paths = os.getenv('PATH', os.defpath).split(os.pathsep)
  145. for path in paths:
  146. for ex in (basename, basename + '.exe'):
  147. fpath = os.path.join(path, ex)
  148. if os.path.exists(fpath) and os.access(fpath, os.X_OK):
  149. return fpath