/papermill/clientwrap.py

https://github.com/nteract/papermill · Python · 110 lines · 61 code · 15 blank · 34 comment · 9 complexity · 3b8f56ac76c6b77298240869471db8d6 MD5 · raw file

  1. import sys
  2. import asyncio
  3. from nbclient import NotebookClient
  4. from nbclient.exceptions import CellExecutionError
  5. from traitlets import Bool, Instance
  6. class PapermillNotebookClient(NotebookClient):
  7. """
  8. Module containing a that executes the code cells
  9. and updates outputs
  10. """
  11. log_output = Bool(False).tag(config=True)
  12. stdout_file = Instance(object, default_value=None).tag(config=True)
  13. stderr_file = Instance(object, default_value=None).tag(config=True)
  14. def __init__(self, nb_man, km=None, raise_on_iopub_timeout=True, **kw):
  15. """Initializes the execution manager.
  16. Parameters
  17. ----------
  18. nb_man : NotebookExecutionManager
  19. Notebook execution manager wrapper being executed.
  20. km : KernerlManager (optional)
  21. Optional kernel manager. If none is provided, a kernel manager will
  22. be created.
  23. """
  24. super().__init__(nb_man.nb, km=km, raise_on_iopub_timeout=raise_on_iopub_timeout, **kw)
  25. self.nb_man = nb_man
  26. def execute(self, **kwargs):
  27. """
  28. Wraps the parent class process call slightly
  29. """
  30. self.reset_execution_trackers()
  31. # See https://bugs.python.org/issue37373 :(
  32. if sys.version_info[0] == 3 and sys.version_info[1] >= 8 and sys.platform.startswith('win'):
  33. asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
  34. with self.setup_kernel(**kwargs):
  35. self.log.info("Executing notebook with kernel: %s" % self.kernel_name)
  36. self.papermill_execute_cells()
  37. info_msg = self.wait_for_reply(self.kc.kernel_info())
  38. self.nb.metadata['language_info'] = info_msg['content']['language_info']
  39. self.set_widgets_metadata()
  40. return self.nb
  41. def papermill_execute_cells(self):
  42. """
  43. This function replaces cell execution with it's own wrapper.
  44. We are doing this for the following reasons:
  45. 1. Notebooks will stop executing when they encounter a failure but not
  46. raise a `CellException`. This allows us to save the notebook with the
  47. traceback even though a `CellExecutionError` was encountered.
  48. 2. We want to write the notebook as cells are executed. We inject our
  49. logic for that here.
  50. 3. We want to include timing and execution status information with the
  51. metadata of each cell.
  52. """
  53. # Execute each cell and update the output in real time.
  54. for index, cell in enumerate(self.nb.cells):
  55. try:
  56. self.nb_man.cell_start(cell, index)
  57. self.execute_cell(cell, index)
  58. except CellExecutionError as ex:
  59. self.nb_man.cell_exception(self.nb.cells[index], cell_index=index, exception=ex)
  60. break
  61. finally:
  62. self.nb_man.cell_complete(self.nb.cells[index], cell_index=index)
  63. def log_output_message(self, output):
  64. """
  65. Process a given output. May log it in the configured logger and/or write it into
  66. the configured stdout/stderr files.
  67. :param output: nbformat.notebooknode.NotebookNode
  68. :return:
  69. """
  70. if output.output_type == "stream":
  71. content = "".join(output.text)
  72. if output.name == "stdout":
  73. if self.log_output:
  74. self.log.info(content)
  75. if self.stdout_file:
  76. self.stdout_file.write(content)
  77. self.stdout_file.flush()
  78. elif output.name == "stderr":
  79. if self.log_output:
  80. # In case users want to redirect stderr differently, pipe to warning
  81. self.log.warning(content)
  82. if self.stderr_file:
  83. self.stderr_file.write(content)
  84. self.stderr_file.flush()
  85. elif self.log_output and ("data" in output and "text/plain" in output.data):
  86. self.log.info("".join(output.data['text/plain']))
  87. def process_message(self, *arg, **kwargs):
  88. output = super().process_message(*arg, **kwargs)
  89. self.nb_man.autosave_cell()
  90. if output and (self.log_output or self.stderr_file or self.stdout_file):
  91. self.log_output_message(output)
  92. return output