/django_logstream/server/backends/zeromq/base.py

https://github.com/niwibe/django-logstream · Python · 144 lines · 104 code · 35 blank · 5 comment · 15 complexity · 771e27246efb764021e124ffc68db72c MD5 · raw file

  1. # -*- coding: utf-8 -*-
  2. from django.conf import settings
  3. from django_logstream.utils import Singleton
  4. from django_logstream.server import storage
  5. from multiprocessing import Process
  6. import zmq
  7. import Queue
  8. import copy
  9. from Crypto.Cipher import Blowfish
  10. from Crypto.Hash import SHA
  11. class WorkerProcessor(object):
  12. def __init__(self):
  13. print "Initializing WorkerProcessor class..."
  14. def __call__(self, queue):
  15. print "Running WorkerProcessor thread..."
  16. self.logrotate_interval = int(getattr(settings,
  17. 'LOGSTREAM_LOGROTATE_INTERVAL', 60))
  18. self.secure_mode = bool(getattr(settings,
  19. 'LOGSTREAM_SECURE_MODE', False))
  20. self.queue = queue
  21. self.storage = storage.Storage(interval=self.logrotate_interval)
  22. self.cipher = Blowfish.new(settings.SECRET_KEY)
  23. while True:
  24. qobj = self.queue.get(block=True)
  25. if not self._analyze_object(qobj):
  26. continue
  27. is_encrypted = bool(qobj.pop('encrypt', False))
  28. valid = False
  29. if is_encrypted:
  30. valid, qobj = self._decrypt(qobj)
  31. if not valid:
  32. continue
  33. # skip all unencrtypted logs on secure
  34. # mode is activated
  35. if self.secure_mode and not valid:
  36. continue
  37. self._process_object(qobj)
  38. def _decrypt(self, obj):
  39. nobj = copy.deepcopy(obj)
  40. valid = True
  41. for key, value in obj.iteritems():
  42. if not isinstance(value, (str, unicode)):
  43. continue
  44. if key.endswith("_sha"):
  45. continue
  46. nobj[key] = self.cipher.decrypt(value).strip("\0")
  47. if SHA.new(nobj[key]).hexdigest() != \
  48. nobj[key + '_sha']:
  49. valid = False
  50. del nobj[key + '_sha']
  51. return valid, nobj
  52. def _analyze_object(self, obj):
  53. if not isinstance(obj, dict):
  54. return False
  55. if 'alias' not in obj:
  56. return False
  57. if 'record' not in obj:
  58. return False
  59. return True
  60. def _process_object(self, obj):
  61. alias, record = obj.pop('alias'), obj.pop('record')
  62. self.storage.insert(alias, record)
  63. class WorkerReceiver(object):
  64. def __init__(self):
  65. print "Initializing WorkerReceiver class..."
  66. def _init(self):
  67. self.context = zmq.Context()
  68. self.bind_address = getattr(settings,
  69. 'LOGSTREAM_BIND_ADDR', 'ipc:///tmp/logstream_receiver')
  70. self.queue = Queue.Queue()
  71. def __call__(self):
  72. print "Running WorkerReceiver process..."
  73. self._init()
  74. self.socket = self.context.socket(zmq.PULL)
  75. self.socket.bind(self.bind_address)
  76. # Starting record processor thread.
  77. from threading import Thread
  78. self.worker = Thread(target=WorkerProcessor(), args=[self.queue])
  79. self.worker.daemon = True
  80. self.worker.start()
  81. # Record receiver loop.
  82. while True:
  83. data = self.socket.recv_pyobj()
  84. self.queue.put_nowait(data)
  85. def __del__(self):
  86. self.worker.terminate()
  87. self.socket.close()
  88. from django.core.exceptions import ImproperlyConfigured
  89. class Backend(object):
  90. __metaclass__ = Singleton
  91. def __init__(self):
  92. self._check_settings()
  93. self.start_receiver()
  94. self.start_rpc()
  95. def _check_settings(self):
  96. try:
  97. getattr(settings, 'LOGSTREAM_STORAGE_PATH')
  98. except:
  99. raise ImproperlyConfigured('LOGSTREAM_STORAGE_PATH is not defined on yout settings')
  100. def start_receiver(self):
  101. self.process = Process(target=WorkerReceiver())
  102. self.process.daemon = True
  103. self.process.start()
  104. def start_rpc(self):
  105. pass
  106. def wait(self):
  107. self.process.join()
  108. def terminate(self):
  109. self.process.terminate()