pytest-notebook by example

Note

This tutorial is a notebook, that can be downloaded to run locally tutorial_intro.ipynb, and was converted to documentation with ipypublish

Python API

The principal component of pytest-notebook is the NBRegressionFixture class, which is an attrs class, whose parameters can be instatiated or set via attibutes.

In:

import importlib_resources
from pytest_notebook import example_nbs
from pytest_notebook.nb_regression import NBRegressionFixture

In:

fixture = NBRegressionFixture(exec_timeout=50)
fixture.diff_color_words = False
fixture

Out:

NBRegressionFixture(exec_notebook=True, exec_cwd=None, exec_allow_errors=False, exec_timeout=50, coverage=False, cov_config=None, cov_source=None, cov_merge=None, post_processors=('coalesce_streams',), process_resources={}, diff_replace=(), diff_ignore=('/cells/*/outputs/*/traceback',), diff_use_color=True, diff_color_words=False, force_regen=False)

The main method is check(), which executes a notebook and compares its initial and final contents.

In:

with importlib_resources.path(example_nbs, "example1.ipynb") as path:
    fixture.check(str(path))

---------------------------------------------------------------------------
NBRegressionError                         Traceback (most recent call last)
<ipython-input-3-23ab8b792066> in <module>
      1 with importlib_resources.path(example_nbs, "example1.ipynb") as path:
----> 2     fixture.check(str(path))

~/GitHub/pytest-notebook/pytest_notebook/nb_regression.py in check(self, path, raise_errors)
    367             raise regen_exc
    368         elif filtered_diff:
--> 369             raise NBRegressionError(diff_string)
    370
    371         return NBRegressionResult(

NBRegressionError:
--- expected
+++ obtained
## replaced /cells/1/execution_count:
-  2
+  1

## modified /cells/2/outputs/0/text:
@@ -1,2 +1,2 @@
 hallo1
-hallo3
+hallo2

To return the results, without raising an exception, use raise_errors=False. This returns a NBRegressionResult instance.

In:

with importlib_resources.path(example_nbs, "example1.ipynb") as path:
    result = fixture.check(str(path), raise_errors=False)

result

Out:

NBRegressionResult(diff_full_length=1,diff_filtered_length=1)

In:

print(result.diff_string)


--- expected
+++ obtained
## replaced /cells/1/execution_count:
-  2
+  1

## modified /cells/2/outputs/0/text:
@@ -1,2 +1,2 @@
 hallo1
-hallo3
+hallo2

The diff of the notebooks is returned as a list of nbdime DiffEntry objects.

In:

result.diff_filtered

Out:

[{'op': 'patch',
  'key': 'cells',
  'diff': [{'op': 'patch',
    'key': 1,
    'diff': [{'op': 'replace', 'key': 'execution_count', 'value': 1}]},
   {'op': 'patch',
    'key': 2,
    'diff': [{'op': 'patch',
      'key': 'outputs',
      'diff': [{'op': 'patch',
        'key': 0,
        'diff': [{'op': 'patch',
          'key': 'text',
          'diff': [{'op': 'patch',
            'key': 1,
            'diff': [{'op': 'addrange', 'key': 5, 'valuelist': '2'},
             {'op': 'removerange', 'key': 5, 'length': 1}]}]}]}]}]}]}]

The notebooks are returned as nbformat.NotebookNode instances.

In:

result.nb_final

Out:

{'cells': [{'cell_type': 'markdown',
   'metadata': {},
   'source': '# Markdown Cell'},
  {'cell_type': 'code',
   'execution_count': 1,
   'metadata': {},
   'outputs': [],
   'source': 'from IPython import display'},
  {'cell_type': 'code',
   'execution_count': 2,
   'metadata': {},
   'outputs': [{'output_type': 'stream',
     'name': 'stdout',
     'text': 'hallo1\nhallo2\n'}],
   'source': "print('hallo1')\nprint('hallo2')"}],
 'metadata': {'kernelspec': {'display_name': 'Python 3',
   'language': 'python',
   'name': 'python3'},
  'language_info': {'name': 'python',
   'version': '3.6.7',
   'mimetype': 'text/x-python',
   'codemirror_mode': {'name': 'ipython', 'version': 3},
   'pygments_lexer': 'ipython3',
   'nbconvert_exporter': 'python',
   'file_extension': '.py'}},
 'nbformat': 4,
 'nbformat_minor': 2}

pytest fixture

See also

pytest_notebook.ipy_magic, for the notebook magic used to run pytest in a Jupyter Notebook.

In:

%load_ext pytest_notebook.ipy_magic

A NBRegressionFixture instance can accessed via the nb_regression() fixture. This instance will be instatiated with parameters dictated by arguments parsed from the pytest command-line and configuration file(s).

Note

pytest-notebook command-line and configuration file parameter names are the same as for NBRegressionFixture, but with the prefix nb_.

The command-line parameter takes precedence over the configuration file one.

In:

%%pytest -v --color=yes --disable-warnings --nb-exec-timeout 50

---
[pytest]
nb_diff_color_words = True
---

import importlib_resources
from pytest_notebook import example_nbs

def test_notebook(nb_regression):
    with importlib_resources.path(example_nbs, "example1.ipynb") as path:
        nb_regression.check(str(path))

============================= test session starts ==============================
platform darwin -- Python 3.6.7, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 -- //anaconda/envs/pytest_nb/bin/python
cachedir: .pytest_cache
rootdir: /private/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmppyh8kyts, inifile: pytest.ini
plugins: cov-2.7.1, datadir-1.3.0, regressions-1.0.5, notebook-0.5.1
collecting ... collected 1 item

test_ipycell.py::test_notebook FAILED                                    [100%]

=================================== FAILURES ===================================
________________________________ test_notebook _________________________________

nb_regression = NBRegressionFixture(exec_notebook=True, exec_cwd='/Users/cjs14/GitHub/pytest-notebook/pytest_notebook/example_nbs', ex...place=(), diff_ignore=('/cells/*/outputs/*/traceback',), diff_use_color=True, diff_color_words=True, force_regen=False)

    def test_notebook(nb_regression):
        with importlib_resources.path(example_nbs, "example1.ipynb") as path:
>           nb_regression.check(str(path))
E           pytest_notebook.nb_regression.NBRegressionError: 
E           --- expected
E           +++ obtained
E           ## replaced /cells/1/execution_count:
E           -  2
E           +  1
E           
E           ## modified /cells/2/outputs/0/text:
E           @@ -1,2 +1,2 @@
E           hallo1
E           hallo3hallo2
E           
E           

test_ipycell.py:8: NBRegressionError
===================== 1 failed, 1 warnings in 2.42 seconds =====================

pytest file collection

check() can be run automatically on all notebooks using the pytest collection mechanism. To activate this feature, set --nb-test-files on the command-line, or nb_test_files = True in the configuration file.

In:

notebook1_content = importlib_resources.read_text(example_nbs, "example1_pass.ipynb")
notebook2_content = importlib_resources.read_text(example_nbs, "example1.ipynb")

In:

%%pytest -v --color=yes --disable-warnings --nb-test-files

***
(notebook1_content, "test_notebook1.ipynb")
(notebook2_content, "test_notebook2.ipynb")
***

============================= test session starts ==============================
platform darwin -- Python 3.6.7, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 -- //anaconda/envs/pytest_nb/bin/python
cachedir: .pytest_cache
rootdir: /private/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmpvs4w3kxu
plugins: cov-2.7.1, datadir-1.3.0, regressions-1.0.5, notebook-0.5.1
collecting ... collected 2 items

test_notebook1.ipynb::nbregression(test_notebook1) PASSED                [ 50%]
test_notebook2.ipynb::nbregression(test_notebook2) FAILED                [100%]

=================================== FAILURES ===================================
____________________ notebook: nbregression(test_notebook2) ____________________
pytest_notebook.nb_regression.NBRegressionError:
--- expected
+++ obtained
## replaced /cells/1/execution_count:
-  2
+  1

## modified /cells/2/outputs/0/text:
@@ -1,2 +1,2 @@
 hallo1
-hallo3
+hallo2

================ 1 failed, 1 passed, 1 warnings in 3.64 seconds ================

In:

%%pytest -v --color=yes --disable-warnings

---
[pytest]
nb_test_files = True
---

***
(notebook1_content, "test_notebook1.ipynb")
(notebook2_content, "test_notebook2.ipynb")
***

============================= test session starts ==============================
platform darwin -- Python 3.6.7, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 -- //anaconda/envs/pytest_nb/bin/python
cachedir: .pytest_cache
rootdir: /private/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmphhv85_77, inifile: pytest.ini
plugins: cov-2.7.1, datadir-1.3.0, regressions-1.0.5, notebook-0.5.1
collecting ... collected 2 items

test_notebook1.ipynb::nbregression(test_notebook1) PASSED                [ 50%]
test_notebook2.ipynb::nbregression(test_notebook2) FAILED                [100%]

=================================== FAILURES ===================================
____________________ notebook: nbregression(test_notebook2) ____________________
pytest_notebook.nb_regression.NBRegressionError:
--- expected
+++ obtained
## replaced /cells/1/execution_count:
-  2
+  1

## modified /cells/2/outputs/0/text:
@@ -1,2 +1,2 @@
 hallo1
-hallo3
+hallo2

================ 1 failed, 1 passed, 1 warnings in 3.76 seconds ================

To restrict the notebook files pytest collects, one or more filename pattern matches can also be set (see fnmatch).

In:

%%pytest -v --color=yes --disable-warnings

---
[pytest]
nb_test_files = True
nb_file_fnmatch = test_*.ipynb tutorial_*.ipynb
---

***
(notebook1_content, "test_notebook1.ipynb")
(notebook2_content, "other_notebook2.ipynb")
***

============================= test session starts ==============================
platform darwin -- Python 3.6.7, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 -- //anaconda/envs/pytest_nb/bin/python
cachedir: .pytest_cache
rootdir: /private/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmpd18whmsj, inifile: pytest.ini
plugins: cov-2.7.1, datadir-1.3.0, regressions-1.0.5, notebook-0.5.1
collecting ... collected 1 item

test_notebook1.ipynb::nbregression(test_notebook1) PASSED                [100%]

===================== 1 passed, 1 warnings in 2.31 seconds =====================

Live Logging of Cell Execution

If you wish to view the progress of the notebook execution, you can use the pytest live-logging functionality:

In:

%%pytest -v --color=yes --disable-warnings --nb-test-files --log-cli-level=info

***
(notebook1_content, "test_notebook1.ipynb")
***

============================= test session starts ==============================
platform darwin -- Python 3.6.7, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 -- //anaconda/envs/pytest_nb/bin/python
cachedir: .pytest_cache
rootdir: /private/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmpzax121qh
plugins: cov-2.7.1, datadir-1.3.0, regressions-1.0.5, notebook-0.5.1
collecting ... collected 1 item

test_notebook1.ipynb::nbregression(test_notebook1)
-------------------------------- live log call ---------------------------------
INFO     pytest_notebook.execution:execution.py:142 About to execute notebook with 3 cells
INFO     pytest_notebook.execution:execution.py:148 Executing notebook with kernel: python3
INFO     pytest_notebook.execution:execution.py:168 Executing cell 1
INFO     pytest_notebook.execution:execution.py:168 Executing cell 2
PASSED                                                                   [100%]

===================== 1 passed, 1 warnings in 2.40 seconds =====================

Regenerating Notebooks

Failing notebooks can be regenerated by setting --nb-force-regen. This will overwrite failing notebooks with the output from the notebook execution.

Note

Notebooks will not be regenerated if they raise any unexpected exceptions, during execution.

This approach to regeneration mimics pytest-regressions.

In:

%%pytest -v --color=yes --disable-warnings --nb-force-regen

---
[pytest]
nb_test_files = True
nb_force_regen = True
---

***
(notebook1_content, "test_notebook1.ipynb")
(notebook2_content, "test_notebook2.ipynb")
***

============================= test session starts ==============================
platform darwin -- Python 3.6.7, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 -- //anaconda/envs/pytest_nb/bin/python
cachedir: .pytest_cache
NB force regen: True
rootdir: /private/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmp7n8y_fg0, inifile: pytest.ini
plugins: cov-2.7.1, datadir-1.3.0, regressions-1.0.5, notebook-0.5.1
collecting ... collected 2 items

test_notebook1.ipynb::nbregression(test_notebook1) PASSED                [ 50%]
test_notebook2.ipynb::nbregression(test_notebook2) FAILED                [100%]

=================================== FAILURES ===================================
____________________ notebook: nbregression(test_notebook2) ____________________
pytest_notebook.nb_regression.NBRegressionError: Files differ and --nb-force-regen set, regenerating file at:
- /private/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmp7n8y_fg0/test_notebook2.ipynb
----------------------------- Captured stderr call -----------------------------
Diff before regeneration:

--- expected
+++ obtained
## replaced /cells/1/execution_count:
-  2
+  1

## modified /cells/2/outputs/0/text:
@@ -1,2 +1,2 @@
 hallo1
-hallo3
+hallo2

================ 1 failed, 1 passed, 1 warnings in 3.67 seconds ================

The regenration can be observed, if we run two tests on the same notebook.

In:

%%pytest -v --color=yes --disable-warnings

import os, tempfile
import importlib_resources
import pytest
from pytest_notebook import example_nbs

@pytest.fixture(scope="module")
def notebook():
    tmphandle, tmppath = tempfile.mkstemp(suffix=".ipynb")
    with open(tmppath, "w") as handle:
        handle.write(
            importlib_resources.read_text(example_nbs, "example1.ipynb")
        )
    yield tmppath
    os.remove(tmppath)

def test_notebook1(nb_regression, notebook):
    nb_regression.force_regen = True
    nb_regression.check(notebook)

def test_notebook2(nb_regression, notebook):
    nb_regression.check(notebook)

============================= test session starts ==============================
platform darwin -- Python 3.6.7, pytest-5.0.1, py-1.8.0, pluggy-0.12.0 -- //anaconda/envs/pytest_nb/bin/python
cachedir: .pytest_cache
rootdir: /private/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmpeb7sjuqh
plugins: cov-2.7.1, datadir-1.3.0, regressions-1.0.5, notebook-0.5.1
collecting ... collected 2 items

test_ipycell.py::test_notebook1 FAILED                                   [ 50%]
test_ipycell.py::test_notebook2 PASSED                                   [100%]

=================================== FAILURES ===================================
________________________________ test_notebook1 ________________________________

nb_regression = NBRegressionFixture(exec_notebook=True, exec_cwd='/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T', exec_allow_errors...place=(), diff_ignore=('/cells/*/outputs/*/traceback',), diff_use_color=True, diff_color_words=False, force_regen=True)
notebook = '/var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmp64orgem_.ipynb'

    def test_notebook1(nb_regression, notebook):
        nb_regression.force_regen = True
>       nb_regression.check(notebook)
E       pytest_notebook.nb_regression.NBRegressionError: Files differ and --nb-force-regen set, regenerating file at:
E       - /var/folders/dm/b2qnkb_n3r72slmpxlfmcjvm00lbnd/T/tmp64orgem_.ipynb

test_ipycell.py:20: NBRegressionError
----------------------------- Captured stderr call -----------------------------
Diff before regeneration:

--- expected
+++ obtained
## replaced /cells/1/execution_count:
-  2
+  1

## modified /cells/2/outputs/0/text:
@@ -1,2 +1,2 @@
 hallo1
-hallo3
+hallo2

================ 1 failed, 1 passed, 1 warnings in 3.76 seconds ================