"""pytest plugin configuration.
For more information on writing pytest plugins see:
- https://docs.pytest.org/en/latest/writing_plugins.html
- https://docs.pytest.org/en/latest/reference.html#request
- https://docs.pytest.org/en/latest/example/simple.html
- https://github.com/pytest-dev/cookiecutter-pytest-plugin
- http://doc.pytest.org/en/latest/example/nonpython.html
- https://docs.pytest.org/en/latest/_modules/_pytest/hookspec.html
"""
import fnmatch
from pathlib import Path
import shlex
from nbclient.exceptions import CellExecutionError
import pytest
from pytest_notebook.diffing import load_nbdime_ignore_config
from pytest_notebook.execution import HELP_EXEC_ENV
from pytest_notebook.nb_regression import (
DEFAULT_DIFF_IGNORE,
HELP_COVERAGE,
HELP_DIFF_COLOR_WORDS,
HELP_DIFF_IGNORE,
HELP_DIFF_REPLACE,
HELP_DIFF_USE_COLOR,
HELP_EXEC_ALLOW_ERRORS,
HELP_EXEC_CWD,
HELP_EXEC_NOTEBOOK,
HELP_EXEC_TIMEOUT,
HELP_FORCE_REGEN,
HELP_POST_PROCS,
NBRegressionFixture,
)
from pytest_notebook.notebook import load_notebook_with_config, validate_regex_replace
HELP_TEST_FILES = "Treat each .ipynb file as a test to be run."
HELP_FILE_FNMATCH = (
"The fnmatch pattern(s) for collecting notebooks, default: '*.ipynb'."
)
HELP_DIFF_USE_NBDIME_CONFIG = (
"Also load diff-ignore paths from an nbdime configuration file "
"(nbdime_config.json), looked up in the current working directory, "
"then the pytest root directory."
)
[docs]
class NotSet:
"""Class to indicate a configuration value was not set."""
[docs]
def pytest_addoption(parser):
"""Add pytest commandline and configuration file options."""
group = parser.getgroup("nb_regression")
group.addoption(
"--nb-test-files",
action="store_true",
default=None,
dest="nb_test_files",
help=HELP_TEST_FILES,
)
# group.addoption(
# "--nb-file-fnmatch", dest="nb_file_fnmatch", type=str, help=HELP_FILE_FNMATCH
# )
# TODO option for --nb-no-exec-notebook
group.addoption("--nb-exec-cwd", dest="nb_exec_cwd", type=str, help=HELP_EXEC_CWD)
group.addoption(
"--nb-exec-errors",
action="store_true",
default=None,
dest="nb_exec_allow_errors",
help=HELP_EXEC_ALLOW_ERRORS,
)
group.addoption(
"--nb-exec-timeout", dest="nb_exec_timeout", type=int, help=HELP_EXEC_TIMEOUT
)
group.addoption(
"--nb-coverage",
action="store_true",
default=None,
dest="nb_coverage",
help=HELP_COVERAGE,
)
group.addoption(
"--nb-diff-color-words",
action="store_true",
default=None,
dest="nb_diff_color_words",
help=HELP_DIFF_COLOR_WORDS,
)
group.addoption(
"--nb-force-regen",
action="store_true",
default=None,
dest="nb_force_regen",
help=HELP_FORCE_REGEN,
)
parser.addini("nb_test_files", type="bool", help=HELP_TEST_FILES, default=NotSet())
parser.addini(
"nb_file_fnmatch", type="args", help=HELP_FILE_FNMATCH, default=NotSet()
)
parser.addini(
"nb_exec_notebook", type="bool", help=HELP_EXEC_NOTEBOOK, default=NotSet()
)
parser.addini("nb_exec_cwd", help=HELP_EXEC_CWD, default=NotSet())
parser.addini(
"nb_exec_allow_errors",
type="bool",
help=HELP_EXEC_ALLOW_ERRORS,
default=NotSet(),
)
parser.addini("nb_exec_timeout", help=HELP_EXEC_TIMEOUT, default=NotSet())
parser.addini("nb_exec_env", type="linelist", help=HELP_EXEC_ENV, default=NotSet())
parser.addini("nb_coverage", type="bool", help=HELP_COVERAGE, default=NotSet())
parser.addini(
"nb_post_processors", type="linelist", help=HELP_POST_PROCS, default=NotSet()
)
parser.addini(
"nb_diff_ignore", type="linelist", help=HELP_DIFF_IGNORE, default=NotSet()
)
parser.addini(
"nb_diff_use_nbdime_config",
type="bool",
help=HELP_DIFF_USE_NBDIME_CONFIG,
default=NotSet(),
)
parser.addini(
"nb_diff_replace", type="linelist", help=HELP_DIFF_REPLACE, default=NotSet()
)
parser.addini(
"nb_diff_use_color", type="bool", help=HELP_DIFF_USE_COLOR, default=NotSet()
)
parser.addini(
"nb_diff_color_words", type="bool", help=HELP_DIFF_COLOR_WORDS, default=NotSet()
)
parser.addini(
"nb_force_regen", type="bool", help=HELP_FORCE_REGEN, default=NotSet()
)
[docs]
def str2bool(string):
"""Convert a string representation of truth to True or False.
True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values
are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if
'val' is anything else.
"""
if isinstance(string, bool):
return string
_map = {
"y": True,
"yes": True,
"t": True,
"true": True,
"on": True,
"1": True,
"n": False,
"no": False,
"f": False,
"false": False,
"off": False,
"0": False,
}
try:
return _map[string.lower()]
except KeyError:
raise ValueError(f'"{string}" is not a valid bool value')
[docs]
def strip_quotes(string):
"""Strip quotes from string."""
if string.startswith("'"):
string = string.strip("'")
elif string.startswith('"'):
string = string.strip('"')
return string
[docs]
def validate_diff_replace(pytestconfig):
r"""Extract the ``nb_diff_replace`` option from the ini file.
This should be of the format::
nb_diff_replace =
/cells/*/outputs \d{1,2}/\d{1,2}/\d{2,4} REPLACE_DATE
/cells/*/outputs "\d{2}:\d{2}:\d{2}" "REPLACE_TIME"
"""
nb_diff_replace = pytestconfig.getini("nb_diff_replace")
if isinstance(nb_diff_replace, NotSet):
return None
if not isinstance(nb_diff_replace, (list, tuple)):
raise ValueError("nb_diff_replace option should be a list or tuple")
output = []
for i, line in enumerate(nb_diff_replace):
args = tuple(strip_quotes(arg) for arg in shlex.split(line, posix=False))
validate_regex_replace(args, i)
output.append(args)
return tuple(output)
[docs]
def validate_exec_env(pytestconfig):
"""Extract the ``nb_exec_env`` option from the ini file.
This should be a list of lines of the format ``KEY=VALUE``::
nb_exec_env =
MY_VAR=my_value
PYTHONPATH=src
"""
nb_exec_env = pytestconfig.getini("nb_exec_env")
if isinstance(nb_exec_env, NotSet):
return None
env = {}
for line in nb_exec_env:
key, sep, value = line.partition("=")
if not sep or not key.strip():
raise ValueError(
f"nb_exec_env line is not of the format KEY=VALUE: '{line}'"
)
env[key.strip()] = value
return env
NBDIME_IGNORE_STASH_KEY = pytest.StashKey()
[docs]
def gather_nbdime_ignore_config(pytestconfig):
"""Load diff-ignore paths from an ``nbdime_config.json`` file.
The (first) file is looked up in the current working directory,
then the pytest root directory, and the result is cached for the session.
"""
if NBDIME_IGNORE_STASH_KEY in pytestconfig.stash:
return pytestconfig.stash[NBDIME_IGNORE_STASH_KEY]
directories = [Path.cwd()]
if pytestconfig.rootpath != directories[0]:
directories.append(pytestconfig.rootpath)
for directory in directories:
config_file = directory / "nbdime_config.json"
if config_file.is_file():
try:
ignores = load_nbdime_ignore_config(config_file)
except Exception as exc:
raise pytest.UsageError(
f"Invalid nbdime configuration file '{config_file}': {exc}"
) from exc
pytestconfig.stash[NBDIME_IGNORE_STASH_KEY] = ignores
return ignores
locations = " or ".join(str(directory) for directory in directories)
raise pytest.UsageError(
f"nb_diff_use_nbdime_config is set, but no nbdime_config.json found in: "
f"{locations}"
)
[docs]
def gather_config_options(pytestconfig):
"""Gather all options, from command-line and ini file.
Note: command-line set options are prioritised over ini file ones.
"""
nbreg_kwargs = {}
for name, value_type in [
("nb_exec_notebook", str2bool),
("nb_exec_cwd", str),
("nb_exec_allow_errors", str2bool),
("nb_exec_timeout", int),
("nb_coverage", str2bool),
("nb_post_processors", tuple),
("nb_diff_ignore", tuple),
("nb_diff_use_color", str2bool),
("nb_diff_color_words", str2bool),
("nb_force_regen", str2bool),
]:
if pytestconfig.getoption(name, None) is not None:
nbreg_kwargs[name[3:]] = value_type(pytestconfig.getoption(name))
elif not isinstance(pytestconfig.getini(name), NotSet):
nbreg_kwargs[name[3:]] = value_type(pytestconfig.getini(name))
other_args = {}
for name, value_type in [("nb_test_files", bool), ("nb_file_fnmatch", tuple)]:
if pytestconfig.getoption(name, None) is not None:
other_args[name] = value_type(pytestconfig.getoption(name))
elif not isinstance(pytestconfig.getini(name), NotSet):
other_args[name] = value_type(pytestconfig.getini(name))
nb_diff_replace = validate_diff_replace(pytestconfig)
if nb_diff_replace is not None:
nbreg_kwargs["diff_replace"] = nb_diff_replace
nb_exec_env = validate_exec_env(pytestconfig)
if nb_exec_env is not None:
nbreg_kwargs["exec_env"] = nb_exec_env
use_nbdime_config = pytestconfig.getini("nb_diff_use_nbdime_config")
if not isinstance(use_nbdime_config, NotSet) and str2bool(use_nbdime_config):
nbreg_kwargs["diff_ignore"] = tuple(
sorted(
set(nbreg_kwargs.get("diff_ignore", DEFAULT_DIFF_IGNORE))
| set(gather_nbdime_ignore_config(pytestconfig))
)
)
# options from pytest_cov
# see: https://github.com/pytest-dev/pytest-cov/blob/master/src/pytest_cov/plugin.py
if pytestconfig.getoption("cov_source", None) is not None:
if pytestconfig.getoption("cov_source") == [""]:
# this is returned if --cov= is used
nbreg_kwargs["cov_source"] = None
else:
nbreg_kwargs["cov_source"] = tuple(pytestconfig.getoption("cov_source"))
if pytestconfig.getoption("cov_config", None) is not None:
nbreg_kwargs["cov_config"] = pytestconfig.getoption("cov_config")
if pytestconfig.pluginmanager.hasplugin("_cov"):
plugin = pytestconfig.pluginmanager.getplugin("_cov")
if plugin.cov_controller:
nbreg_kwargs["cov_merge"] = plugin.cov_controller.cov
return nbreg_kwargs, other_args
[docs]
@pytest.fixture(scope="function")
def nb_regression(pytestconfig):
"""Fixture to execute a Jupyter Notebook, and test its output is as expected."""
kwargs, _ = gather_config_options(pytestconfig)
return NBRegressionFixture(**kwargs)
[docs]
def _matches_pattern(file_path: Path, pattern: str) -> bool:
"""Match a file against an fnmatch pattern.
Patterns containing a path separator are matched against the full path
(relative patterns match any trailing path segments),
otherwise against the file name (mirroring ``py.path.local.fnmatch``).
"""
if "/" in pattern:
if not pattern.startswith("/"):
pattern = "*/" + pattern
return fnmatch.fnmatch(file_path.as_posix(), pattern)
return fnmatch.fnmatch(file_path.name, pattern)
[docs]
def pytest_collect_file(file_path: Path, parent):
"""Collect Jupyter notebooks using the specified pytest hook."""
_, other_args = gather_config_options(parent.config)
if other_args.get("nb_test_files", False) and any(
_matches_pattern(file_path, pat)
for pat in other_args.get("nb_file_fnmatch", ["*.ipynb"])
):
return JupyterNbCollector.from_parent(parent, path=file_path)
[docs]
class JupyterNbCollector(pytest.File):
"""This class represents a pytest collector object for Jupyter Notebook files.
A collector is associated with a .ipynb file and collects it for testing.
"""
[docs]
def collect(self):
"""Collect tests for the notebook."""
yield JupyterNbTest.from_parent(self, name=f"nbregression({self.path.stem})")
[docs]
class JupyterNbTest(pytest.Item):
"""This class represents a pytest test invocation for a Jupyter Notebook file."""
def __init__(self, name, parent):
"""Initialise the class, parsing the notebook metadata, and adding markers."""
super().__init__(name, parent)
self._fixtureinfo = self.session._fixturemanager.getfixtureinfo(
self.parent, NBRegressionFixture.check, NBRegressionFixture
) # this is required for --setup-plan
_, nb_config = load_notebook_with_config(str(self.path))
if nb_config.skip:
self.add_marker(pytest.mark.skip(reason=nb_config.skip_reason))
[docs]
def runtest(self):
"""Run the test."""
kwargs, _ = gather_config_options(self.config)
fixture = NBRegressionFixture(**kwargs)
try:
fixture.check(str(self.path))
except CellExecutionError as err:
# allow a notebook to skip itself at runtime,
# by raising ``pytest.skip(...)`` within a cell
if err.ename == "Skipped":
pytest.skip(err.evalue)
raise
[docs]
def repr_failure(self, exc_info):
"""Handle exception raised by ``self.runtest()``.
:param exc_info: see
https://docs.pytest.org/en/latest/reference.html#_pytest._code.ExceptionInfo
"""
# return exc_info.getrepr()
return exc_info.exconly()
[docs]
def reportinfo(self):
"""Report location of item."""
return self.path, 0, f"notebook: {self.name}"