/Win32/Lib/multiprocessing/connection.py

https://gitlab.com/CoastHeavyIndustries/OSM-stats · Python · 428 lines · 291 code · 70 blank · 67 comment · 61 complexity · 6b92b2fdfe4d982db3f32207a01f473f MD5 · raw file

  1. #
  2. # A higher level module for using sockets (or Windows named pipes)
  3. #
  4. # multiprocessing/connection.py
  5. #
  6. # Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
  7. #
  8. __all__ = [ 'Client', 'Listener', 'Pipe' ]
  9. import os
  10. import sys
  11. import socket
  12. import errno
  13. import time
  14. import tempfile
  15. import itertools
  16. import _multiprocessing
  17. from multiprocessing import current_process, AuthenticationError
  18. from multiprocessing.util import get_temp_dir, Finalize, sub_debug, debug
  19. from multiprocessing.forking import duplicate, close
  20. #
  21. #
  22. #
  23. BUFSIZE = 8192
  24. # A very generous timeout when it comes to local connections...
  25. CONNECTION_TIMEOUT = 20.
  26. _mmap_counter = itertools.count()
  27. default_family = 'AF_INET'
  28. families = ['AF_INET']
  29. if hasattr(socket, 'AF_UNIX'):
  30. default_family = 'AF_UNIX'
  31. families += ['AF_UNIX']
  32. if sys.platform == 'win32':
  33. default_family = 'AF_PIPE'
  34. families += ['AF_PIPE']
  35. def _init_timeout(timeout=CONNECTION_TIMEOUT):
  36. return time.time() + timeout
  37. def _check_timeout(t):
  38. return time.time() > t
  39. #
  40. #
  41. #
  42. def arbitrary_address(family):
  43. '''
  44. Return an arbitrary free address for the given family
  45. '''
  46. if family == 'AF_INET':
  47. return ('localhost', 0)
  48. elif family == 'AF_UNIX':
  49. return tempfile.mktemp(prefix='listener-', dir=get_temp_dir())
  50. elif family == 'AF_PIPE':
  51. return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
  52. (os.getpid(), _mmap_counter.next()))
  53. else:
  54. raise ValueError('unrecognized family')
  55. def address_type(address):
  56. '''
  57. Return the types of the address
  58. This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE'
  59. '''
  60. if type(address) == tuple:
  61. return 'AF_INET'
  62. elif type(address) is str and address.startswith('\\\\'):
  63. return 'AF_PIPE'
  64. elif type(address) is str:
  65. return 'AF_UNIX'
  66. else:
  67. raise ValueError('address type of %r unrecognized' % address)
  68. #
  69. # Public functions
  70. #
  71. class Listener(object):
  72. '''
  73. Returns a listener object.
  74. This is a wrapper for a bound socket which is 'listening' for
  75. connections, or for a Windows named pipe.
  76. '''
  77. def __init__(self, address=None, family=None, backlog=1, authkey=None):
  78. family = family or (address and address_type(address)) \
  79. or default_family
  80. address = address or arbitrary_address(family)
  81. if family == 'AF_PIPE':
  82. self._listener = PipeListener(address, backlog)
  83. else:
  84. self._listener = SocketListener(address, family, backlog)
  85. if authkey is not None and not isinstance(authkey, bytes):
  86. raise TypeError, 'authkey should be a byte string'
  87. self._authkey = authkey
  88. def accept(self):
  89. '''
  90. Accept a connection on the bound socket or named pipe of `self`.
  91. Returns a `Connection` object.
  92. '''
  93. c = self._listener.accept()
  94. if self._authkey:
  95. deliver_challenge(c, self._authkey)
  96. answer_challenge(c, self._authkey)
  97. return c
  98. def close(self):
  99. '''
  100. Close the bound socket or named pipe of `self`.
  101. '''
  102. return self._listener.close()
  103. address = property(lambda self: self._listener._address)
  104. last_accepted = property(lambda self: self._listener._last_accepted)
  105. def Client(address, family=None, authkey=None):
  106. '''
  107. Returns a connection to the address of a `Listener`
  108. '''
  109. family = family or address_type(address)
  110. if family == 'AF_PIPE':
  111. c = PipeClient(address)
  112. else:
  113. c = SocketClient(address)
  114. if authkey is not None and not isinstance(authkey, bytes):
  115. raise TypeError, 'authkey should be a byte string'
  116. if authkey is not None:
  117. answer_challenge(c, authkey)
  118. deliver_challenge(c, authkey)
  119. return c
  120. if sys.platform != 'win32':
  121. def Pipe(duplex=True):
  122. '''
  123. Returns pair of connection objects at either end of a pipe
  124. '''
  125. if duplex:
  126. s1, s2 = socket.socketpair()
  127. c1 = _multiprocessing.Connection(os.dup(s1.fileno()))
  128. c2 = _multiprocessing.Connection(os.dup(s2.fileno()))
  129. s1.close()
  130. s2.close()
  131. else:
  132. fd1, fd2 = os.pipe()
  133. c1 = _multiprocessing.Connection(fd1, writable=False)
  134. c2 = _multiprocessing.Connection(fd2, readable=False)
  135. return c1, c2
  136. else:
  137. from _multiprocessing import win32
  138. def Pipe(duplex=True):
  139. '''
  140. Returns pair of connection objects at either end of a pipe
  141. '''
  142. address = arbitrary_address('AF_PIPE')
  143. if duplex:
  144. openmode = win32.PIPE_ACCESS_DUPLEX
  145. access = win32.GENERIC_READ | win32.GENERIC_WRITE
  146. obsize, ibsize = BUFSIZE, BUFSIZE
  147. else:
  148. openmode = win32.PIPE_ACCESS_INBOUND
  149. access = win32.GENERIC_WRITE
  150. obsize, ibsize = 0, BUFSIZE
  151. h1 = win32.CreateNamedPipe(
  152. address, openmode,
  153. win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |
  154. win32.PIPE_WAIT,
  155. 1, obsize, ibsize, win32.NMPWAIT_WAIT_FOREVER, win32.NULL
  156. )
  157. h2 = win32.CreateFile(
  158. address, access, 0, win32.NULL, win32.OPEN_EXISTING, 0, win32.NULL
  159. )
  160. win32.SetNamedPipeHandleState(
  161. h2, win32.PIPE_READMODE_MESSAGE, None, None
  162. )
  163. try:
  164. win32.ConnectNamedPipe(h1, win32.NULL)
  165. except WindowsError, e:
  166. if e.args[0] != win32.ERROR_PIPE_CONNECTED:
  167. raise
  168. c1 = _multiprocessing.PipeConnection(h1, writable=duplex)
  169. c2 = _multiprocessing.PipeConnection(h2, readable=duplex)
  170. return c1, c2
  171. #
  172. # Definitions for connections based on sockets
  173. #
  174. class SocketListener(object):
  175. '''
  176. Representation of a socket which is bound to an address and listening
  177. '''
  178. def __init__(self, address, family, backlog=1):
  179. self._socket = socket.socket(getattr(socket, family))
  180. self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
  181. self._socket.bind(address)
  182. self._socket.listen(backlog)
  183. self._address = self._socket.getsockname()
  184. self._family = family
  185. self._last_accepted = None
  186. if family == 'AF_UNIX':
  187. self._unlink = Finalize(
  188. self, os.unlink, args=(address,), exitpriority=0
  189. )
  190. else:
  191. self._unlink = None
  192. def accept(self):
  193. s, self._last_accepted = self._socket.accept()
  194. fd = duplicate(s.fileno())
  195. conn = _multiprocessing.Connection(fd)
  196. s.close()
  197. return conn
  198. def close(self):
  199. self._socket.close()
  200. if self._unlink is not None:
  201. self._unlink()
  202. def SocketClient(address):
  203. '''
  204. Return a connection object connected to the socket given by `address`
  205. '''
  206. family = address_type(address)
  207. s = socket.socket( getattr(socket, family) )
  208. t = _init_timeout()
  209. while 1:
  210. try:
  211. s.connect(address)
  212. except socket.error, e:
  213. if e.args[0] != errno.ECONNREFUSED or _check_timeout(t):
  214. debug('failed to connect to address %s', address)
  215. raise
  216. time.sleep(0.01)
  217. else:
  218. break
  219. else:
  220. raise
  221. fd = duplicate(s.fileno())
  222. conn = _multiprocessing.Connection(fd)
  223. s.close()
  224. return conn
  225. #
  226. # Definitions for connections based on named pipes
  227. #
  228. if sys.platform == 'win32':
  229. class PipeListener(object):
  230. '''
  231. Representation of a named pipe
  232. '''
  233. def __init__(self, address, backlog=None):
  234. self._address = address
  235. handle = win32.CreateNamedPipe(
  236. address, win32.PIPE_ACCESS_DUPLEX,
  237. win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |
  238. win32.PIPE_WAIT,
  239. win32.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
  240. win32.NMPWAIT_WAIT_FOREVER, win32.NULL
  241. )
  242. self._handle_queue = [handle]
  243. self._last_accepted = None
  244. sub_debug('listener created with address=%r', self._address)
  245. self.close = Finalize(
  246. self, PipeListener._finalize_pipe_listener,
  247. args=(self._handle_queue, self._address), exitpriority=0
  248. )
  249. def accept(self):
  250. newhandle = win32.CreateNamedPipe(
  251. self._address, win32.PIPE_ACCESS_DUPLEX,
  252. win32.PIPE_TYPE_MESSAGE | win32.PIPE_READMODE_MESSAGE |
  253. win32.PIPE_WAIT,
  254. win32.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
  255. win32.NMPWAIT_WAIT_FOREVER, win32.NULL
  256. )
  257. self._handle_queue.append(newhandle)
  258. handle = self._handle_queue.pop(0)
  259. try:
  260. win32.ConnectNamedPipe(handle, win32.NULL)
  261. except WindowsError, e:
  262. if e.args[0] != win32.ERROR_PIPE_CONNECTED:
  263. raise
  264. return _multiprocessing.PipeConnection(handle)
  265. @staticmethod
  266. def _finalize_pipe_listener(queue, address):
  267. sub_debug('closing listener with address=%r', address)
  268. for handle in queue:
  269. close(handle)
  270. def PipeClient(address):
  271. '''
  272. Return a connection object connected to the pipe given by `address`
  273. '''
  274. t = _init_timeout()
  275. while 1:
  276. try:
  277. win32.WaitNamedPipe(address, 1000)
  278. h = win32.CreateFile(
  279. address, win32.GENERIC_READ | win32.GENERIC_WRITE,
  280. 0, win32.NULL, win32.OPEN_EXISTING, 0, win32.NULL
  281. )
  282. except WindowsError, e:
  283. if e.args[0] not in (win32.ERROR_SEM_TIMEOUT,
  284. win32.ERROR_PIPE_BUSY) or _check_timeout(t):
  285. raise
  286. else:
  287. break
  288. else:
  289. raise
  290. win32.SetNamedPipeHandleState(
  291. h, win32.PIPE_READMODE_MESSAGE, None, None
  292. )
  293. return _multiprocessing.PipeConnection(h)
  294. #
  295. # Authentication stuff
  296. #
  297. MESSAGE_LENGTH = 20
  298. CHALLENGE = b'#CHALLENGE#'
  299. WELCOME = b'#WELCOME#'
  300. FAILURE = b'#FAILURE#'
  301. def deliver_challenge(connection, authkey):
  302. import hmac
  303. assert isinstance(authkey, bytes)
  304. message = os.urandom(MESSAGE_LENGTH)
  305. connection.send_bytes(CHALLENGE + message)
  306. digest = hmac.new(authkey, message).digest()
  307. response = connection.recv_bytes(256) # reject large message
  308. if response == digest:
  309. connection.send_bytes(WELCOME)
  310. else:
  311. connection.send_bytes(FAILURE)
  312. raise AuthenticationError('digest received was wrong')
  313. def answer_challenge(connection, authkey):
  314. import hmac
  315. assert isinstance(authkey, bytes)
  316. message = connection.recv_bytes(256) # reject large message
  317. assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
  318. message = message[len(CHALLENGE):]
  319. digest = hmac.new(authkey, message).digest()
  320. connection.send_bytes(digest)
  321. response = connection.recv_bytes(256) # reject large message
  322. if response != WELCOME:
  323. raise AuthenticationError('digest sent was rejected')
  324. #
  325. # Support for using xmlrpclib for serialization
  326. #
  327. class ConnectionWrapper(object):
  328. def __init__(self, conn, dumps, loads):
  329. self._conn = conn
  330. self._dumps = dumps
  331. self._loads = loads
  332. for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'):
  333. obj = getattr(conn, attr)
  334. setattr(self, attr, obj)
  335. def send(self, obj):
  336. s = self._dumps(obj)
  337. self._conn.send_bytes(s)
  338. def recv(self):
  339. s = self._conn.recv_bytes()
  340. return self._loads(s)
  341. def _xml_dumps(obj):
  342. return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf8')
  343. def _xml_loads(s):
  344. (obj,), method = xmlrpclib.loads(s.decode('utf8'))
  345. return obj
  346. class XmlListener(Listener):
  347. def accept(self):
  348. global xmlrpclib
  349. import xmlrpclib
  350. obj = Listener.accept(self)
  351. return ConnectionWrapper(obj, _xml_dumps, _xml_loads)
  352. def XmlClient(*args, **kwds):
  353. global xmlrpclib
  354. import xmlrpclib
  355. return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads)