Update tutorial CI workflow to ignore NotImplementedError exceptions (N…

…euromatchAcademy#542)

* Update tutorial CI workflow to ignore NotImplementedError exceptions

* Update commit message extraction per github deprecation

* Clean up docstring

* Clarify origin of custom preprocessor and location of changes

* Move custom preprocessor class to after main function

* Fix typo

.github/workflows/notebook-pr.yaml

@@ -24,7 +24,7 @@ jobs:
       - name: Get commit message
         run: |
           readonly local msg=$(git log -1 --pretty=format:"%s")
-          echo "::set-env name=COMMIT_MESSAGE::$msg"
+          echo "COMMIT_MESSAGE=$msg" >> $GITHUB_ENV
 
       - name: Set up Python
         if: "!contains(env.COMMIT_MESSAGE, 'skip ci')"

ci/process_notebooks.py

@@ -4,7 +4,7 @@
 - Check that the cells have been executed sequentially on a fresh kernel
 - Strip trailing whitespace from all code lines
 - Execute the notebook and fail if errors are encountered
-- Extract solution code and write a .py file witht the solution
+- Extract solution code and write a .py file with the solution
 - Replace solution cells with a "hint" image and a link to the solution code
 - Redirect Colab-inserted badges to the master branch
 - Set the Colab notebook name field based on file path
@@ -21,12 +21,17 @@
 import sys
 import argparse
 import hashlib
+import typing as t
 from io import BytesIO
 from binascii import a2b_base64
-from PIL import Image
 from copy import deepcopy
+import asyncio
+
+from PIL import Image
 import nbformat
 from nbconvert.preprocessors import ExecutePreprocessor
+from nbclient.util import run_sync, ensure_async
+from nbclient.exceptions import CellControlSignal, CellExecutionError, DeadKernelError
 
 
 GITHUB_RAW_URL = (
@@ -73,7 +78,7 @@ def should_process(path):
             nb = nbformat.read(f, nbformat.NO_CONVERT)
 
         if not sequentially_executed(nb):
-            if args.require_sequntial:
+            if args.require_sequential:
                 err = (
                     "Notebook is not sequentially executed on a fresh kernel."
                     "\n"
@@ -87,7 +92,7 @@ def should_process(path):
 
         # Run the notebook from top to bottom, catching errors
         print(f"Executing {nb_path}")
-        executor = ExecutePreprocessor(**exec_kws)
+        executor = NMAPreprocessor(**exec_kws)
         try:
             executor.preprocess(nb)
         except Exception as err:
@@ -165,6 +170,183 @@ def should_process(path):
     exit(errors)
 
 
+# ------------------------------------------------------------------------------------ #
+
+
+class NMAPreprocessor(ExecutePreprocessor):
+    """
+    Custom subclass of the ExecutePreprocessor for NMA tutorials.
+
+    This class overwrites the execute_cell method to ignore NotImplementedError
+    exceptions, which are raised when incomplete exercise functions are called.
+    All other errors will be handled as normal.
+
+    """
+    # Note: we have to patch the entire async_execute_cell method because it checks
+    # for errors with a private method (_check_raise_for_error). It would be cleaner
+    # to customize only the error handling method, but alas, that is not allowed.
+
+    # The nbconvert.ExecutePreprocessor class inherits from both
+    # nbconvert.Preprocessor and nbclient.NotebookClient. The relevant methods that we
+    # patch are defined on the latter. The code here was taken from this specific tag:
+    # https://github.com/jupyter/nbclient/blob/0.5.1/nbclient/client.py
+
+    async def async_execute_cell(
+            self,
+            cell: nbformat.NotebookNode,
+            cell_index: int,
+            execution_count: t.Optional[int] = None,
+            store_history: bool = True) -> nbformat.NotebookNode:
+        """
+        Executes a single code cell.
+
+        To execute all cells see :meth:`execute`.
+
+        Parameters
+        ----------
+        cell : nbformat.NotebookNode
+            The cell which is currently being processed.
+        cell_index : int
+            The position of the cell within the notebook object.
+        execution_count : int
+            The execution count assigned to the cell (default: Use kernel response)
+        store_history : bool
+            Determines if history should be stored in the kernel (default: False).
+            Specific to ipython kernels, which can store command histories.
+
+        Raises
+        ------
+        CellExecutionError
+            If execution failed and should raise an exception, this will be raised
+            with defaults about the failure.
+
+        Returns
+        -------
+        cell : NotebookNode
+            The cell which was just processed.
+
+        License
+        -------
+
+        This project is licensed under the terms of the Modified BSD License
+        (also known as New or Revised or 3-Clause BSD), as follows:
+
+        - Copyright (c) 2020-, Jupyter Development Team
+
+        All rights reserved.
+
+        Redistribution and use in source and binary forms, with or without
+        modification, are permitted provided that the following conditions are met:
+
+        Redistributions of source code must retain the above copyright notice, this
+        list of conditions and the following disclaimer.
+
+        Redistributions in binary form must reproduce the above copyright notice, this
+        list of conditions and the following disclaimer in the documentation and/or
+        other materials provided with the distribution.
+
+        Neither the name of the Jupyter Development Team nor the names of its
+        contributors may be used to endorse or promote products derived from this
+        software without specific prior written permission.
+
+        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+        ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+        WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+        DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+        """
+        assert self.kc is not None
+        if cell.cell_type != 'code' or not cell.source.strip():
+            self.log.debug("Skipping non-executing cell %s", cell_index)
+            return cell
+
+        if self.record_timing and 'execution' not in cell['metadata']:
+            cell['metadata']['execution'] = {}
+
+        self.log.debug("Executing cell:\n%s", cell.source)
+        parent_msg_id = await ensure_async(
+            self.kc.execute(
+                cell.source,
+                store_history=store_history,
+                stop_on_error=not self.allow_errors
+            )
+        )
+        # We launched a code cell to execute
+        self.code_cells_executed += 1
+        exec_timeout = self._get_timeout(cell)
+
+        cell.outputs = []
+        self.clear_before_next_output = False
+
+        task_poll_kernel_alive = asyncio.ensure_future(
+            self._async_poll_kernel_alive()
+        )
+        task_poll_output_msg = asyncio.ensure_future(
+            self._async_poll_output_msg(parent_msg_id, cell, cell_index)
+        )
+        self.task_poll_for_reply = asyncio.ensure_future(
+            self._async_poll_for_reply(
+                parent_msg_id,
+                cell,
+                exec_timeout,
+                task_poll_output_msg,
+                task_poll_kernel_alive,
+            )
+        )
+        try:
+            exec_reply = await self.task_poll_for_reply
+        except asyncio.CancelledError:
+            # can only be cancelled by task_poll_kernel_alive when the kernel is dead
+            task_poll_output_msg.cancel()
+            raise DeadKernelError("Kernel died")
+        except Exception as e:
+            # Best effort to cancel request if it hasn't been resolved
+            try:
+                # Check if the task_poll_output is doing the raising for us
+                if not isinstance(e, CellControlSignal):
+                    task_poll_output_msg.cancel()
+            finally:
+                raise
+
+        if execution_count:
+            cell['execution_count'] = execution_count
+
+        # -- NMA-specific code here -- #
+        self._check_raise_for_error_nma(cell, exec_reply)
+
+        self.nb['cells'][cell_index] = cell
+        return cell
+
+    def _check_raise_for_error_nma(
+            self,
+            cell: nbformat.NotebookNode,
+            exec_reply: t.Optional[t.Dict]) -> None:
+
+        cell_tags = cell.metadata.get("tags", [])
+        cell_allows_errors = self.allow_errors or "raises-exception" in cell_tags
+
+        if self.force_raise_errors or not cell_allows_errors:
+
+            if (exec_reply is not None) and exec_reply['content']['status'] == 'error':
+
+                # -- NMA-specific code here -- #
+                if exec_reply['content']['ename'] != 'NotImplementedError':
+
+                    raise CellExecutionError.from_cell_and_msg(cell,
+                                                               exec_reply['content'])
+
+    execute_cell = run_sync(async_execute_cell)
+
+
+# ------------------------------------------------------------------------------------ #
+
+
 def extract_solutions(nb, nb_dir, nb_name):
     """Convert solution cells to markdown; embed images from Python output."""
     nb = deepcopy(nb)
@@ -396,7 +578,7 @@ def parse_args(arglist):
     parser.add_argument(
         "--allow-non-sequential",
         action="store_false",
-        dest="require_sequntial",
+        dest="require_sequential",
         help="Don't fail if the notebook is not sequentially executed"
     )
     return parser.parse_args(arglist)