/Lib/multiprocessing/forking.py

http://unladen-swallow.googlecode.com/ · Python · 473 lines · 299 code · 89 blank · 85 comment · 77 complexity · 43f64ead7d054b639715b9cb512451bd MD5 · raw file

  1. #
  2. # Module for starting a process object using os.fork() or CreateProcess()
  3. #
  4. # multiprocessing/forking.py
  5. #
  6. # Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt
  7. #
  8. import os
  9. import sys
  10. import signal
  11. from multiprocessing import util, process
  12. __all__ = ['Popen', 'assert_spawning', 'exit', 'duplicate', 'close', 'ForkingPickler']
  13. #
  14. # Check that the current thread is spawning a child process
  15. #
  16. def assert_spawning(self):
  17. if not Popen.thread_is_spawning():
  18. raise RuntimeError(
  19. '%s objects should only be shared between processes'
  20. ' through inheritance' % type(self).__name__
  21. )
  22. #
  23. # Try making some callable types picklable
  24. #
  25. from pickle import Pickler
  26. class ForkingPickler(Pickler):
  27. dispatch = Pickler.dispatch.copy()
  28. @classmethod
  29. def register(cls, type, reduce):
  30. def dispatcher(self, obj):
  31. rv = reduce(obj)
  32. self.save_reduce(obj=obj, *rv)
  33. cls.dispatch[type] = dispatcher
  34. def _reduce_method(m):
  35. if m.im_self is None:
  36. return getattr, (m.im_class, m.im_func.func_name)
  37. else:
  38. return getattr, (m.im_self, m.im_func.func_name)
  39. ForkingPickler.register(type(ForkingPickler.save), _reduce_method)
  40. def _reduce_method_descriptor(m):
  41. return getattr, (m.__objclass__, m.__name__)
  42. ForkingPickler.register(type(list.append), _reduce_method_descriptor)
  43. ForkingPickler.register(type(int.__add__), _reduce_method_descriptor)
  44. #def _reduce_builtin_function_or_method(m):
  45. # return getattr, (m.__self__, m.__name__)
  46. #ForkingPickler.register(type(list().append), _reduce_builtin_function_or_method)
  47. #ForkingPickler.register(type(int().__add__), _reduce_builtin_function_or_method)
  48. try:
  49. from functools import partial
  50. except ImportError:
  51. pass
  52. else:
  53. def _reduce_partial(p):
  54. return _rebuild_partial, (p.func, p.args, p.keywords or {})
  55. def _rebuild_partial(func, args, keywords):
  56. return partial(func, *args, **keywords)
  57. ForkingPickler.register(partial, _reduce_partial)
  58. #
  59. # Unix
  60. #
  61. if sys.platform != 'win32':
  62. import time
  63. exit = os._exit
  64. duplicate = os.dup
  65. close = os.close
  66. #
  67. # We define a Popen class similar to the one from subprocess, but
  68. # whose constructor takes a process object as its argument.
  69. #
  70. class Popen(object):
  71. def __init__(self, process_obj):
  72. sys.stdout.flush()
  73. sys.stderr.flush()
  74. self.returncode = None
  75. self.pid = os.fork()
  76. if self.pid == 0:
  77. if 'random' in sys.modules:
  78. import random
  79. random.seed()
  80. code = process_obj._bootstrap()
  81. sys.stdout.flush()
  82. sys.stderr.flush()
  83. os._exit(code)
  84. def poll(self, flag=os.WNOHANG):
  85. if self.returncode is None:
  86. pid, sts = os.waitpid(self.pid, flag)
  87. if pid == self.pid:
  88. if os.WIFSIGNALED(sts):
  89. self.returncode = -os.WTERMSIG(sts)
  90. else:
  91. assert os.WIFEXITED(sts)
  92. self.returncode = os.WEXITSTATUS(sts)
  93. return self.returncode
  94. def wait(self, timeout=None):
  95. if timeout is None:
  96. return self.poll(0)
  97. deadline = time.time() + timeout
  98. delay = 0.0005
  99. while 1:
  100. res = self.poll()
  101. if res is not None:
  102. break
  103. remaining = deadline - time.time()
  104. if remaining <= 0:
  105. break
  106. delay = min(delay * 2, remaining, 0.05)
  107. time.sleep(delay)
  108. return res
  109. def terminate(self):
  110. if self.returncode is None:
  111. try:
  112. os.kill(self.pid, signal.SIGTERM)
  113. except OSError, e:
  114. if self.wait(timeout=0.1) is None:
  115. raise
  116. @staticmethod
  117. def thread_is_spawning():
  118. return False
  119. #
  120. # Windows
  121. #
  122. else:
  123. import thread
  124. import msvcrt
  125. import _subprocess
  126. import time
  127. from ._multiprocessing import win32, Connection, PipeConnection
  128. from .util import Finalize
  129. #try:
  130. # from cPickle import dump, load, HIGHEST_PROTOCOL
  131. #except ImportError:
  132. from pickle import load, HIGHEST_PROTOCOL
  133. def dump(obj, file, protocol=None):
  134. ForkingPickler(file, protocol).dump(obj)
  135. #
  136. #
  137. #
  138. TERMINATE = 0x10000
  139. WINEXE = (sys.platform == 'win32' and getattr(sys, 'frozen', False))
  140. exit = win32.ExitProcess
  141. close = win32.CloseHandle
  142. #
  143. # _python_exe is the assumed path to the python executable.
  144. # People embedding Python want to modify it.
  145. #
  146. if sys.executable.lower().endswith('pythonservice.exe'):
  147. _python_exe = os.path.join(sys.exec_prefix, 'python.exe')
  148. else:
  149. _python_exe = sys.executable
  150. def set_executable(exe):
  151. global _python_exe
  152. _python_exe = exe
  153. #
  154. #
  155. #
  156. def duplicate(handle, target_process=None, inheritable=False):
  157. if target_process is None:
  158. target_process = _subprocess.GetCurrentProcess()
  159. return _subprocess.DuplicateHandle(
  160. _subprocess.GetCurrentProcess(), handle, target_process,
  161. 0, inheritable, _subprocess.DUPLICATE_SAME_ACCESS
  162. ).Detach()
  163. #
  164. # We define a Popen class similar to the one from subprocess, but
  165. # whose constructor takes a process object as its argument.
  166. #
  167. class Popen(object):
  168. '''
  169. Start a subprocess to run the code of a process object
  170. '''
  171. _tls = thread._local()
  172. def __init__(self, process_obj):
  173. # create pipe for communication with child
  174. rfd, wfd = os.pipe()
  175. # get handle for read end of the pipe and make it inheritable
  176. rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True)
  177. os.close(rfd)
  178. # start process
  179. cmd = get_command_line() + [rhandle]
  180. cmd = ' '.join('"%s"' % x for x in cmd)
  181. hp, ht, pid, tid = _subprocess.CreateProcess(
  182. _python_exe, cmd, None, None, 1, 0, None, None, None
  183. )
  184. ht.Close()
  185. close(rhandle)
  186. # set attributes of self
  187. self.pid = pid
  188. self.returncode = None
  189. self._handle = hp
  190. # send information to child
  191. prep_data = get_preparation_data(process_obj._name)
  192. to_child = os.fdopen(wfd, 'wb')
  193. Popen._tls.process_handle = int(hp)
  194. try:
  195. dump(prep_data, to_child, HIGHEST_PROTOCOL)
  196. dump(process_obj, to_child, HIGHEST_PROTOCOL)
  197. finally:
  198. del Popen._tls.process_handle
  199. to_child.close()
  200. @staticmethod
  201. def thread_is_spawning():
  202. return getattr(Popen._tls, 'process_handle', None) is not None
  203. @staticmethod
  204. def duplicate_for_child(handle):
  205. return duplicate(handle, Popen._tls.process_handle)
  206. def wait(self, timeout=None):
  207. if self.returncode is None:
  208. if timeout is None:
  209. msecs = _subprocess.INFINITE
  210. else:
  211. msecs = max(0, int(timeout * 1000 + 0.5))
  212. res = _subprocess.WaitForSingleObject(int(self._handle), msecs)
  213. if res == _subprocess.WAIT_OBJECT_0:
  214. code = _subprocess.GetExitCodeProcess(self._handle)
  215. if code == TERMINATE:
  216. code = -signal.SIGTERM
  217. self.returncode = code
  218. return self.returncode
  219. def poll(self):
  220. return self.wait(timeout=0)
  221. def terminate(self):
  222. if self.returncode is None:
  223. try:
  224. _subprocess.TerminateProcess(int(self._handle), TERMINATE)
  225. except WindowsError:
  226. if self.wait(timeout=0.1) is None:
  227. raise
  228. #
  229. #
  230. #
  231. def is_forking(argv):
  232. '''
  233. Return whether commandline indicates we are forking
  234. '''
  235. if len(argv) >= 2 and argv[1] == '--multiprocessing-fork':
  236. assert len(argv) == 3
  237. return True
  238. else:
  239. return False
  240. def freeze_support():
  241. '''
  242. Run code for process object if this in not the main process
  243. '''
  244. if is_forking(sys.argv):
  245. main()
  246. sys.exit()
  247. def get_command_line():
  248. '''
  249. Returns prefix of command line used for spawning a child process
  250. '''
  251. if process.current_process()._identity==() and is_forking(sys.argv):
  252. raise RuntimeError('''
  253. Attempt to start a new process before the current process
  254. has finished its bootstrapping phase.
  255. This probably means that you are on Windows and you have
  256. forgotten to use the proper idiom in the main module:
  257. if __name__ == '__main__':
  258. freeze_support()
  259. ...
  260. The "freeze_support()" line can be omitted if the program
  261. is not going to be frozen to produce a Windows executable.''')
  262. if getattr(sys, 'frozen', False):
  263. return [sys.executable, '--multiprocessing-fork']
  264. else:
  265. prog = 'from multiprocessing.forking import main; main()'
  266. return [_python_exe, '-c', prog, '--multiprocessing-fork']
  267. def main():
  268. '''
  269. Run code specifed by data received over pipe
  270. '''
  271. assert is_forking(sys.argv)
  272. handle = int(sys.argv[-1])
  273. fd = msvcrt.open_osfhandle(handle, os.O_RDONLY)
  274. from_parent = os.fdopen(fd, 'rb')
  275. process.current_process()._inheriting = True
  276. preparation_data = load(from_parent)
  277. prepare(preparation_data)
  278. self = load(from_parent)
  279. process.current_process()._inheriting = False
  280. from_parent.close()
  281. exitcode = self._bootstrap()
  282. exit(exitcode)
  283. def get_preparation_data(name):
  284. '''
  285. Return info about parent needed by child to unpickle process object
  286. '''
  287. from .util import _logger, _log_to_stderr
  288. d = dict(
  289. name=name,
  290. sys_path=sys.path,
  291. sys_argv=sys.argv,
  292. log_to_stderr=_log_to_stderr,
  293. orig_dir=process.ORIGINAL_DIR,
  294. authkey=process.current_process().authkey,
  295. )
  296. if _logger is not None:
  297. d['log_level'] = _logger.getEffectiveLevel()
  298. if not WINEXE:
  299. main_path = getattr(sys.modules['__main__'], '__file__', None)
  300. if not main_path and sys.argv[0] not in ('', '-c'):
  301. main_path = sys.argv[0]
  302. if main_path is not None:
  303. if not os.path.isabs(main_path) and \
  304. process.ORIGINAL_DIR is not None:
  305. main_path = os.path.join(process.ORIGINAL_DIR, main_path)
  306. d['main_path'] = os.path.normpath(main_path)
  307. return d
  308. #
  309. # Make (Pipe)Connection picklable
  310. #
  311. def reduce_connection(conn):
  312. if not Popen.thread_is_spawning():
  313. raise RuntimeError(
  314. 'By default %s objects can only be shared between processes\n'
  315. 'using inheritance' % type(conn).__name__
  316. )
  317. return type(conn), (Popen.duplicate_for_child(conn.fileno()),
  318. conn.readable, conn.writable)
  319. ForkingPickler.register(Connection, reduce_connection)
  320. ForkingPickler.register(PipeConnection, reduce_connection)
  321. #
  322. # Prepare current process
  323. #
  324. old_main_modules = []
  325. def prepare(data):
  326. '''
  327. Try to get current process ready to unpickle process object
  328. '''
  329. old_main_modules.append(sys.modules['__main__'])
  330. if 'name' in data:
  331. process.current_process().name = data['name']
  332. if 'authkey' in data:
  333. process.current_process()._authkey = data['authkey']
  334. if 'log_to_stderr' in data and data['log_to_stderr']:
  335. util.log_to_stderr()
  336. if 'log_level' in data:
  337. util.get_logger().setLevel(data['log_level'])
  338. if 'sys_path' in data:
  339. sys.path = data['sys_path']
  340. if 'sys_argv' in data:
  341. sys.argv = data['sys_argv']
  342. if 'dir' in data:
  343. os.chdir(data['dir'])
  344. if 'orig_dir' in data:
  345. process.ORIGINAL_DIR = data['orig_dir']
  346. if 'main_path' in data:
  347. main_path = data['main_path']
  348. main_name = os.path.splitext(os.path.basename(main_path))[0]
  349. if main_name == '__init__':
  350. main_name = os.path.basename(os.path.dirname(main_path))
  351. if main_name != 'ipython':
  352. import imp
  353. if main_path is None:
  354. dirs = None
  355. elif os.path.basename(main_path).startswith('__init__.py'):
  356. dirs = [os.path.dirname(os.path.dirname(main_path))]
  357. else:
  358. dirs = [os.path.dirname(main_path)]
  359. assert main_name not in sys.modules, main_name
  360. file, path_name, etc = imp.find_module(main_name, dirs)
  361. try:
  362. # We would like to do "imp.load_module('__main__', ...)"
  363. # here. However, that would cause 'if __name__ ==
  364. # "__main__"' clauses to be executed.
  365. main_module = imp.load_module(
  366. '__parents_main__', file, path_name, etc
  367. )
  368. finally:
  369. if file:
  370. file.close()
  371. sys.modules['__main__'] = main_module
  372. main_module.__name__ = '__main__'
  373. # Try to make the potentially picklable objects in
  374. # sys.modules['__main__'] realize they are in the main
  375. # module -- somewhat ugly.
  376. for obj in main_module.__dict__.values():
  377. try:
  378. if obj.__module__ == '__parents_main__':
  379. obj.__module__ = '__main__'
  380. except Exception:
  381. pass