diff --git a/readme.ipynb b/readme.ipynb index 60b23ad..c1799fc 100644 --- a/readme.ipynb +++ b/readme.ipynb @@ -112,8 +112,7 @@ "metadata": {}, "outputs": [], "source": [ - " from importnb import Partial\n", - " with Partial():\n", + " with Notebook(exceptions=BaseException):\n", " try: from . import readme\n", " except: import readme" ] @@ -215,8 +214,8 @@ "metadata": {}, "outputs": [], "source": [ - " from importnb import Parameterize\n", - " f = Parameterize(readme)\n", + " from importnb.execute import Parameterize\n", + " f = Parameterize().from_filename(readme.__file__)\n", " " ] }, @@ -374,7 +373,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -390,7 +389,7 @@ "test_imports (src.importnb.tests.test_unittests.TestRemote) ... skipped 'requires IP'\n", "\n", "----------------------------------------------------------------------\n", - "Ran 7 tests in 2.024s\n", + "Ran 7 tests in 2.019s\n", "\n", "OK (skipped=1, expected failures=1)\n" ] @@ -404,26 +403,27 @@ " for path in Path(root).rglob(\"\"\"*.ipynb\"\"\"): \n", " if 'checkpoint' not in str(path):\n", " export(path, Path('src/importnb') / path.with_suffix('.py').relative_to(root))\n", + " \n", " \n", " __import__('unittest').main(module='src.importnb.tests.test_unittests', argv=\"discover --verbose\".split(), exit=False) \n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[NbConvertApp] Converting notebook readme.ipynb to markdown\n" + "[NbConvertApp] Converting notebook readme.ipynb to markdown\n", + "[NbConvertApp] Writing 7383 bytes to readme.md\n" ] } ], "source": [ " if __name__ == '__main__':\n", - " \n", " !jupyter nbconvert --to markdown readme.ipynb" ] }, diff --git a/readme.md b/readme.md index 89108ba..111893f 100644 --- a/readme.md +++ b/readme.md @@ -61,8 +61,7 @@ The [`importnb.loader.Partial`](src/notebooks/loader.ipynb#Partial-Loader) will ```python - from importnb import Partial - with Partial(): + with Notebook(exceptions=BaseException): try: from . import readme except: import readme ``` @@ -118,8 +117,8 @@ In `readme`, `foo` is a parameter because it may be evaluated with ast.literal_v ```python - from importnb import Parameterize - f = Parameterize(readme) + from importnb.execute import Parameterize + f = Parameterize().from_filename(readme.__file__) ``` @@ -223,6 +222,7 @@ For example, create a file called `tricks.yaml` containing for path in Path(root).rglob("""*.ipynb"""): if 'checkpoint' not in str(path): export(path, Path('src/importnb') / path.with_suffix('.py').relative_to(root)) + __import__('unittest').main(module='src.importnb.tests.test_unittests', argv="discover --verbose".split(), exit=False) @@ -237,7 +237,7 @@ For example, create a file called `tricks.yaml` containing test_imports (src.importnb.tests.test_unittests.TestRemote) ... skipped 'requires IP' ---------------------------------------------------------------------- - Ran 7 tests in 1.014s + Ran 7 tests in 1.011s FAILED (skipped=1, unexpected successes=1) @@ -245,12 +245,14 @@ For example, create a file called `tricks.yaml` containing ```python if __name__ == '__main__': - !jupyter nbconvert --to markdown readme.ipynb ``` [NbConvertApp] Converting notebook readme.ipynb to markdown - [NbConvertApp] Writing 7109 bytes to readme.md + [NbConvertApp] Support files will be in readme_files/ + [NbConvertApp] Making directory readme_files + [NbConvertApp] Making directory readme_files + [NbConvertApp] Writing 8946 bytes to readme.md if __name__ == '__main__': diff --git a/src/importnb/__init__.py b/src/importnb/__init__.py index 3909b46..5d3de54 100644 --- a/src/importnb/__init__.py +++ b/src/importnb/__init__.py @@ -1,30 +1,24 @@ # coding: utf-8 -__all__ = "Notebook", "Partial", "reload", "Parameterize", "Lazy", "NotebookTest", "testmod", "Execute" +__all__ = "Notebook", "reload", "Parameterize", "NotebookTest", "testmod", "Execute" try: from .loader import ( Notebook, - Partial, load_ipython_extension, unload_ipython_extension, reload, - Lazy, ) - from .parameterize import Parameterize from .nbtest import NotebookTest, testmod - from .execute import Execute + from .execute import Execute, Parameterize except: from loader import ( Notebook, - Partial, load_ipython_extension, unload_ipython_extension, reload, - Lazy, ) - from parameterize import Parameterize - from execute import Execute + from execute import Execute, Parameterize from nbtest import NotebookTest, testmod \ No newline at end of file diff --git a/src/importnb/capture.py b/src/importnb/capture.py index 0b470e4..6f47f2e 100644 --- a/src/importnb/capture.py +++ b/src/importnb/capture.py @@ -3,18 +3,18 @@ """ try: - from IPython.utils.capture import capture_output + from IPython.utils.capture import capture_output, CapturedIO from IPython import get_ipython assert get_ipython(), """There is no interactive shell""" except: from contextlib import redirect_stdout, ExitStack from io import StringIO + import sys try: from contextlib import redirect_stderr except: - import sys class redirect_stderr: @@ -64,6 +64,14 @@ def stdout(self): def stderr(self): return self._stderr and self._stderr.getvalue() or "" + def show(self): + """write my output to sys.stdout/err as appropriate""" + sys.stdout.write(self.stdout) + sys.stderr.write(self.stderr) + sys.stdout.flush() + sys.stderr.flush() + + if __name__ == "__main__": try: from .utils.export import export diff --git a/src/importnb/decoder.py b/src/importnb/decoder.py index 19818b0..163c08e 100644 --- a/src/importnb/decoder.py +++ b/src/importnb/decoder.py @@ -17,9 +17,11 @@ except: from textwrap import dedent + def identity(*x): return x[0] + class LineNumberDecoder(JSONDecoder): """A JSON Decoder to return a NotebookNode with lines numbers in the metadata.""" @@ -65,74 +67,15 @@ def _parse_object( {"lineno": len(s_and_end[0][:next].rsplit('"source":', 1)[0].splitlines())} ) - if object["cell_type"] == "markdown": - object["source"] = codify_markdown(object["source"]) - object["outputs"] = [] - object["cell_type"] = "code" - object["execution_count"] = None - for key in ("source", "text"): if key in object: object[key] = "".join(object[key]) return object, next -@singledispatch -def codify_markdown(string_or_list): - raise TypeError("Markdown must be a string or a list.") - - -@codify_markdown.register(str) -def codify_markdown_string(str): - if '"""' in str: - str = "'''{}\n'''".format(str) - else: - str = '"""{}\n"""'.format(str) - return str - -@codify_markdown.register(list) -def codify_markdown_list(str): - return list(map("{}\n".format, codify_markdown_string("".join(str)).splitlines())) - - -load = partial(_load, cls=LineNumberDecoder) loads = partial(_loads, cls=LineNumberDecoder) -def cell_to_ast(object, transform=identity, prefix=False): - module = ast.increment_lineno( - ast.parse(transform("".join(object["source"]))), object["metadata"].get("lineno", 1) - ) - prefix and module.body.insert(0, ast.Expr(ast.Ellipsis())) - return module - -def transform_cells(object, transform=dedent): - for cell in object["cells"]: - if "source" in cell: - cell["source"] = transform("".join(cell["source"])) - return object - - -def ast_from_cells(object, transform=identity): - import ast - - module = ast.Module(body=[]) - for cell in object["cells"]: - module.body.extend( - ast.fix_missing_locations( - ast.increment_lineno( - ast.parse("".join(cell["source"])), cell["metadata"].get("lineno", 1) - ) - ).body - ) - return module - -def loads_ast(object, loads=loads, transform=dedent, ast_transform=identity): - if isinstance(object, str): - object = loads(object) - object = transform_cells(object, transform) - return ast_from_cells(object, ast_transform) - if __name__ == "__main__": try: from .utils.export import export @@ -141,4 +84,3 @@ def loads_ast(object, loads=loads, transform=dedent, ast_transform=identity): export("decoder.ipynb", "../decoder.py") __import__("doctest").testmod() - diff --git a/src/importnb/execute.py b/src/importnb/execute.py index 8a2b793..690c69b 100644 --- a/src/importnb/execute.py +++ b/src/importnb/execute.py @@ -10,33 +10,35 @@ An executed notebook contains a `__notebook__` attributes that is populated with cell outputs. - >>> assert nb.__notebook__ + >>> assert nb._notebook The `__notebook__` attribute complies with `nbformat` >>> from nbformat.v4 import new_notebook - >>> assert new_notebook(**nb.__notebook__), """The notebook is not a valid nbformat""" + >>> assert new_notebook(**nb._notebook), """The notebook is not a valid nbformat""" ''' -try: +if globals().get("show", None): + print("I am tested.") +try: from .capture import capture_output - from .loader import Notebook, lazy_loader_cls - from .decoder import loads_ast, identity, loads, dedent, cell_to_ast + from .loader import Notebook, advanced_exec_module + from .decoder import identity, loads, dedent except: from capture import capture_output - from loader import Notebook, lazy_loader_cls - from decoder import loads_ast, identity, loads, dedent, cell_to_ast + from loader import Notebook, advanced_exec_module + from decoder import identity, loads, dedent import inspect, sys, ast from functools import partialmethod, partial from importlib import reload, _bootstrap -from traceback import print_exc, format_exc -from warnings import warn -import traceback +from importlib._bootstrap import _call_with_frames_removed, _new_module -__all__ = "Notebook", "Partial", "reload", "Lazy" +import traceback +from traceback import print_exc, format_exc, format_tb +from pathlib import Path from ast import ( NodeTransformer, @@ -50,6 +52,39 @@ Ellipsis, Interactive, ) +from collections import ChainMap + +__all__ = "Notebook", "Partial", "reload", "Lazy" + +"""# Loaders that reproduce notebook outputs +""" + + +def loader_include_notebook(loader, module): + if not hasattr(module, "_notebook"): + module._notebook = loads(loader.get_data(loader.path).decode("utf-8")) + + +class NotebookCells(Notebook): + """The NotebookCells loader's contain a _notebook attributes containing a state of the notebook. + + >>> assert NotebookCells().from_filename('execute.ipynb', 'importnb.notebooks')._notebook + """ + + @advanced_exec_module + def exec_module(self, module, **globals): + loader_include_notebook(self, module) + _call_with_frames_removed( + exec, self.source_to_code(module._notebook), module.__dict__, module.__dict__ + ) + + +if __name__ == "__main__": + nb = NotebookCells().from_filename("execute.ipynb", "importnb.notebooks") + +"""## Recreating IPython output objectss +""" + def new_stream(text, name="stdout"): return {"name": name, "output_type": "stream", "text": text} @@ -60,118 +95,167 @@ def new_error(Exception): "ename": type(Exception).__name__, "output_type": "error", "evalue": str(Exception), - "traceback": traceback.format_tb(Exception.__traceback__), + "traceback": format_tb(Exception.__traceback__), } def new_display(object): return {"data": object.data, "metadata": {}, "output_type": "display_data"} + +"""# Reproduce notebooks with the `Execute` class. +""" + + class Execute(Notebook): - """A SourceFileLoader for notebooks that provides line number debugginer in the JSON source.""" + """The Execute loader reproduces outputs in the module._notebook attribute. - def create_module(self, spec): - module = super().create_module(spec) - module.__notebook__ = self._loads(self.get_data(self.path).decode("utf-8")) - return module + >>> nb_raw = Notebook(display=True, stdout=True).from_filename('execute.ipynb', 'importnb.notebooks') + >>> with Execute(display=True, stdout=True) as loader: + ... nb = loader.from_filename('execute.ipynb', 'importnb.notebooks', show=True) + + The loader includes the first markdown cell or leading block string as the docstring. + + >>> assert nb.__doc__ and nb_raw.__doc__ + >>> assert nb.__doc__ == nb_raw.__doc__ - def _iter_cells(self, module): - for i, cell in enumerate(module.__notebook__["cells"]): - if cell["cell_type"] == "code": - yield self._compile( - fix_missing_locations( - self.visit(cell_to_ast(cell, transform=self.format, prefix=i > 0)) - ), - self.path or "", - "exec", - ) + Nothing should have been executed. + + >>> assert any(cell.get('outputs', None) for cell in nb._notebook['cells']) + """ + @advanced_exec_module def exec_module(self, module, **globals): - """All exceptions specific in the context. - """ - module.__dict__.update(globals) - for cell in module.__notebook__["cells"]: + # Remove the outputs + loader_include_notebook(self, module) + for cell in module._notebook["cells"]: if "outputs" in cell: cell["outputs"] = [] - for i, code in enumerate(self._iter_cells(module)): + for i, cell in enumerate(module._notebook["cells"]): error = None - with capture_output( - stdout=self.stdout, stderr=self.stderr, display=self.display - ) as out: + with capture_output() as out: try: - _bootstrap._call_with_frames_removed( - exec, code, module.__dict__, module.__dict__ + _call_with_frames_removed( + exec, + self.source_to_code(cell, interactive=bool(i)), + module.__dict__, + module.__dict__, ) - except BaseException as e: - error = new_error(e) - print(error) + except BaseException as Exception: + error = new_error(Exception) try: - module.__exception__ = e - raise e - except self._exceptions: + module.__exception__ = Exception + raise Exception + except self.exceptions: ... break finally: - if out.outputs: - cell["outputs"] += [new_display(object) for object in out.outputs] - if out.stdout: - - cell["outputs"] += [new_stream(out.stdout)] - if error: - cell["outputs"] += [error] - if out.stderr: - cell["outputs"] += [new_stream(out.stderr, "stderr")] - -""" if __name__ == '__main__': - m = Execute(stdout=True).from_filename('loader.ipynb') + if "outputs" in cell: + if out.outputs: + cell["outputs"] += [new_display(object) for object in out.outputs] + if out.stdout: + cell["outputs"] += [new_stream(out.stdout)] + if error: + cell["outputs"] += [error] + if out.stderr: + cell["outputs"] += [new_stream(out.stderr, "stderr")] + out.show() + + def source_to_code(loader, object, path=None, interactive=True): + """Transform ast modules into Interactive and Expression nodes. This + will allow the cell outputs to be captured. `interactive` is only true for + the first markdown cell. + """ + + node = loader.visit(object) + + if isinstance(node, ast.Expr): + """An expression is a special case where a markdown cell was + turned into a docstring + """ + if interactive: + """An expression will not print to the stdout.""" + node = ast.Expression(body=node.value) + mode = "eval" + else: + """This tree replaces the docstring""" + node = ast.Module(body=[node]) + mode = "exec" + elif isinstance(node, ast.Module): + """In the exectuion mode we use interactive nodes to capture stdouts.""" + node = ast.Interactive(body=node.body) + mode = "single" + return compile(node, path or "", mode) + + +if __name__ == "__main__": + nb = Execute().from_filename("execute.ipynb", "importnb.notebooks") + +"""# Parameterizing notebooks """ -class ParameterizeNode(NodeTransformer): - visit_Module = NodeTransformer.generic_visit - def visit_Assign(FreeStatement, node): +class AssignmentFinder(NodeTransformer): + visit_Interactive = visit_Module = NodeTransformer.generic_visit + + def visit_Assign(self, node): if len(node.targets): try: if not getattr(node.targets[0], "id", "_").startswith("_"): literal_eval(node.value) return node except: - assert True, """The target can not will not literally evaluate.""" - return None + ... def generic_visit(self, node): ... -class ExecuteNode(ParameterizeNode): - def visit_Assign(self, node): - if super().visit_Assign(node): - return ast.Expr(Ellipsis()) - return node +class AssignmentIgnore(AssignmentFinder): - def generic_visit(self, node): + def visit_Assign(self, node): + if isinstance(super().visit_Assign(node), ast.Assign): + return ast.Expr(ast.NameConstant(value=None)) return node -def vars_to_sig(**vars): - """Create a signature for a dictionary of names.""" - from inspect import Parameter, Signature + generic_visit = NodeTransformer.generic_visit - return Signature([Parameter(str, Parameter.KEYWORD_ONLY, default=vars[str]) for str in vars]) -from collections import ChainMap - -class Parameterize(Execute, ExecuteNode): +class Parameterize(Execute): + """Discover any literal ast expression and create parameters from them. + + >>> f = Parameterize().from_filename('execute.ipynb', 'importnb.notebooks') + >>> assert 'a_variable_to_parameterize' in f.__signature__.parameters + >>> assert f(a_variable_to_parameterize=100).a_variable_to_parameterize == 100 + + Parametize is a NodeTransformer that import any nodes return by Parameterize Node. + + >>> assert len(Parameterize().visit(ast.parse(''' + ... foo = 42 + ... bar = foo''')).body) ==2 + """ def create_module(self, spec): module = super().create_module(spec) - nodes = self._data_to_ast(module.__notebook__) + + # Import the notebook when parameterize is imported + loader_include_notebook(self, module) + + node = Notebook().visit(module._notebook) + + # Extra effort to supply a docstring doc = None - if isinstance(nodes, ast.Module) and nodes.body: - node = nodes.body[0] - if isinstance(node, ast.Expr) and isinstance(node.value, ast.Str): - doc = node - params = ParameterizeNode().visit(nodes) + if isinstance(node, ast.Module) and node.body: + _node = node.body[0] + if isinstance(_node, ast.Expr) and isinstance(_node.value, ast.Str): + doc = _node + + # Discover the parameterizable nodes + params = AssignmentFinder().visit(node) + # Include the string in the compilation doc and params.body.insert(0, doc) + + # Supply the literal parameter values as module globals. exec(compile(params, "", "exec"), module.__dict__, module.__dict__) return module @@ -189,9 +273,22 @@ def recall(**kwargs): recall.__doc__ = module.__doc__ return recall -""" if __name__ == '__main__': - f = Parameterize().from_filename('execute.ipynb') -""" + def visit(self, node): + return AssignmentIgnore().visit(super().visit(node)) + + +def vars_to_sig(**vars): + """Create a signature for a dictionary of names.""" + from inspect import Parameter, Signature + + return Signature([Parameter(str, Parameter.KEYWORD_ONLY, default=vars[str]) for str in vars]) + + +if __name__ == "__main__": + f = Parameterize(display=True).from_filename("execute.ipynb", "importnb.notebooks") + print(f(a_variable_to_parameterize=1000).a_variable_to_parameterize) + +a_variable_to_parameterize = 42 """# Developer """ @@ -205,6 +302,5 @@ def recall(**kwargs): module = Execute().from_filename("execute.ipynb") __import__("doctest").testmod(module, verbose=2) -"""For more information check out [`importnb`](https://github.com/deathbeds/importnb) +"""How do the interactive nodes work? """ - diff --git a/src/importnb/loader.py b/src/importnb/loader.py index 67eda3c..5615232 100644 --- a/src/importnb/loader.py +++ b/src/importnb/loader.py @@ -9,15 +9,15 @@ ### `importnb.Partial` - >>> with Partial(): + >>> with Notebook(exceptions=BaseException): ... from importnb.notebooks import loader - >>> assert loader.__exception__ is None + >>> assert loader._exception is None ## There is a [lazy importer]() The Lazy importer will delay the module execution until it is used the first time. It is a useful approach for delaying visualization or data loading. - >>> with Lazy(): + >>> with Notebook(lazy=True): ... from importnb.notebooks import loader ## Loading from resources @@ -34,20 +34,25 @@ >>> with Notebook(stdout=True, stderr=True, display=True, globals=dict(show=True)): ... from importnb.notebooks import loader - >>> assert loader.__output__ - - # loader.__output__.stdout + >>> assert loader._capture + +## Assigning globals + + >>> nb = Notebook(stdout=True, globals={'show': True}).from_filename('loader.ipynb', 'importnb.notebooks') + >>> assert nb._capture.stdout """ -if "show" in globals(): +if globals().get("show", None): print("Catch me if you can") try: - from .capture import capture_output - from .decoder import identity, loads, dedent, cell_to_ast + from .capture import capture_output, CapturedIO + from .decoder import identity, loads, dedent + from .path_hooks import PathHooksContext, modify_sys_path, add_path_hooks, remove_one_path_hook except: - from capture import capture_output - from decoder import identity, loads, dedent, cell_to_ast + from capture import capture_output, CapturedIO + from decoder import identity, loads, dedent + from path_hooks import PathHooksContext, modify_sys_path, add_path_hooks, remove_one_path_hook import inspect, sys, ast from copy import copy @@ -73,10 +78,9 @@ def _init_module_attrs(spec, module): from io import StringIO -from functools import partialmethod, partial +from functools import partialmethod, partial, wraps, singledispatch from importlib import reload -from traceback import print_exc, format_exc -from warnings import warn +from traceback import print_exc, format_exc, format_tb from contextlib import contextmanager, ExitStack from pathlib import Path @@ -85,84 +89,35 @@ def _init_module_attrs(spec, module): except: from importlib_resources import path -__all__ = "Notebook", "Partial", "reload", "Lazy" - -"""## `sys.path_hook` modifiers -""" - -@contextmanager -def modify_file_finder_details(): - """yield the FileFinder in the sys.path_hooks that loads Python files and assure - the import cache is cleared afterwards. - - Everything goes to shit if the import cache is not cleared.""" - - for id, hook in enumerate(sys.path_hooks): - try: - closure = inspect.getclosurevars(hook).nonlocals - except TypeError: - continue - if issubclass(closure["cls"], FileFinder): - sys.path_hooks.pop(id) - details = list(closure["loader_details"]) - yield details - break - sys.path_hooks.insert(id, FileFinder.path_hook(*details)) - sys.path_importer_cache.clear() - -"""Update the file_finder details with functions to append and remove the [loader details](https://docs.python.org/3.7/library/importlib.html#importlib.machinery.FileFinder). -""" - -def add_path_hooks(loader: SourceFileLoader, extensions, *, position=0): - """Update the FileFinder loader in sys.path_hooks to accomodate a {loader} with the {extensions}""" - with modify_file_finder_details() as details: - details.insert(position, (loader, extensions)) - - -def remove_one_path_hook(loader): - loader = lazy_loader_cls(loader) - with modify_file_finder_details() as details: - _details = list(details) - for ct, (cls, ext) in enumerate(_details): - cls = lazy_loader_cls(cls) - if cls == loader: - details.pop(ct) - break +from ast import ( + NodeTransformer, + parse, + Assign, + literal_eval, + dump, + fix_missing_locations, + Str, + Tuple, + Ellipsis, + Interactive, +) +from collections import ChainMap -def lazy_loader_cls(loader): - """Extract the loader contents of a lazy loader in the import path.""" - try: - return inspect.getclosurevars(loader).nonlocals.get("cls", loader) - except: - return loader - -"""## Loader Context Manager +__all__ = "Notebook", "Partial", "reload", "Lazy" -`importnb` uses a context manager to assure that the traditional import system behaviors as expected. If the loader is permenantly available then it may create some unexpected import behaviors. -""" class ImportNbException(BaseException): """ImportNbException allows all exceptions to be raised, a null except statement always passes.""" -class PathHooksContext: - def __enter__(self, position=0): - add_path_hooks(self.prepare(self), self.EXTENSION_SUFFIXES, position=position) - return self +"""## Converting cells to code - def __exit__(self, *excepts): - remove_one_path_hook(self) +These functions are attached to the loaders.s +""" - def prepare(self, loader): - if self._lazy: - try: - from importlib.util import LazyLoader +"""## Loading from resources +""" - if self._lazy: - loader = LazyLoader.factory(loader) - except: - ImportWarning("""LazyLoading is only available in > Python 3.5""") - return loader def from_resource(loader, file=None, resource=None, exec=True, **globals): """Load a python module or notebook from a file location. @@ -171,7 +126,7 @@ def from_resource(loader, file=None, resource=None, exec=True, **globals): This still needs some work for packages. - >>> assert from_resource(Notebook(), 'loader.ipynb', 'importnb.notebooks') + >> assert from_resource(Notebook(), 'loader.ipynb', 'importnb.notebooks') """ with ExitStack() as stack: if resource is not None: @@ -199,25 +154,106 @@ def from_resource(loader, file=None, resource=None, exec=True, **globals): module.__loader__.exec_module(module, **globals) return module -@contextmanager -def modify_sys_path(file): - """This is only invoked when using from_resource.""" - path = str(Path(file).parent) - if path not in map(str, map(Path, sys.path)): - yield sys.path.insert(0, path) - sys.path = [object for object in sys.path if str(Path(object)) != path] - else: - yield - -class Notebook(SourceFileLoader, PathHooksContext, capture_output, ast.NodeTransformer): - """A SourceFileLoader for notebooks that provides line number debugginer in the JSON source.""" + +"""# The Notebook Loader +""" + + +class NotebookLoader(SourceFileLoader, PathHooksContext, NodeTransformer): + """The simplest implementation of a Notebook Source File Loader. + >>> with NotebookLoader(): + ... from importnb.notebooks import decoder + >>> assert decoder.__file__.endswith('.ipynb') + """ + EXTENSION_SUFFIXES = ".ipynb", + __slots__ = "name", "path", + + def __init__(self, fullname=None, path=None): + super().__init__(fullname, path) + + format = staticmethod(dedent) + from_filename = from_resource + + def __call__(self, fullname=None, path=None): + self = copy(self) + return SourceFileLoader.__init__(self, str(fullname), str(path)) or self + + def source_to_code(loader, object, path=None): + node = loader.visit(object) + return compile(node, path or "", "exec") + + def visit(self, node, **opts): + if isinstance(node, bytes): + node = loads(node.decode("utf-8")) + + if isinstance(node, dict): + if "cells" in node: + body = [] + for cell in node["cells"]: + _node = self.visit(cell) + _node = ast.increment_lineno(_node, cell["metadata"].get("lineno", 1)) + body.extend(getattr(_node, "body", [_node])) + node = ast.Module(body=body) + + elif "source" in node: + source = "".join(node["source"]) + if node["cell_type"] == "markdown": + node = ast.Expr(ast.Str(s=source)) + elif node["cell_type"] == "code": + node = ast.parse( + self.format(source), self.path or "", "exec" + ) + else: + node = ast.Module(body=[]) + return ast.fix_missing_locations(super().visit(node)) + + +"""## An advanced `exec_module` decorator. +""" + + +def advanced_exec_module(exec_module): + """Decorate `SourceFileLoader.exec_module` objects with abilities to: + * Capture output in Python and IPython + * Prepopulate a model namespace. + * Allow exceptions while notebooks are loading.s + + >>> assert advanced_exec_module(SourceFileLoader.exec_module) + """ + + def _exec_module(loader, module, **globals): + module._exception = None + module.__dict__.update(globals) + with capture_output( + stdout=loader.stdout, stderr=loader.stderr, display=loader.display + ) as out: + module._capture = out + try: + exec_module(loader, module) + except loader.exceptions as Exception: + module._exception = Exception + + return _exec_module + + +"""# The Advanced Notebook loader +""" + + +class Notebook(NotebookLoader, capture_output): + """The Notebook loader is an advanced loader for IPython notebooks: + + * Capture stdout, stderr, and display objects. + * Partially evaluate notebook with known exceptions. + * Supply extra global values into notebooks. + + >>> assert Notebook().from_filename('loader.ipynb', 'importnb.notebooks') + """ EXTENSION_SUFFIXES = ".ipynb", - _compile = staticmethod(compile) - _loads = staticmethod(loads) format = _transform = staticmethod(dedent) - __slots__ = "stdout", "stderr", "display", "_lazy", "_exceptions", "globals" + __slots__ = "stdout", "stderr", "display", "_lazy", "exceptions", "globals" def __init__( self, @@ -228,107 +264,23 @@ def __init__( stderr=False, display=False, lazy=False, - exceptions=ImportNbException, - globals=None + globals=None, + exceptions=ImportNbException ): - SourceFileLoader.__init__(self, fullname, path) + super().__init__(fullname, path) capture_output.__init__(self, stdout=stdout, stderr=stderr, display=display) self._lazy = lazy - self._exceptions = exceptions self.globals = {} if globals is None else globals - - def __call__(self, fullname=None, path=None): - self = copy(self) - return SourceFileLoader.__init__(self, str(fullname), str(path)) or self - - def visit(self, node): - node = super().visit(node) - return ast.fix_missing_locations(super().visit(node)) + self.exceptions = exceptions def create_module(self, spec): module = _new_module(spec.name) _init_module_attrs(spec, module) - module.__exception__ = None module.__dict__.update(self.globals) return module - def exec_module(self, module, **globals): - """All exceptions specific in the context. - """ - module.__dict__.update(globals) - with capture_output(stdout=self.stdout, stderr=self.stderr, display=self.display) as out: - module.__output__ = out - try: - super().exec_module(module) - except self._exceptions as e: - """Display a message if an error is escaped.""" - module.__exception__ = e - warn( - ".".join( - [ - """{name} was partially imported with a {error}""".format( - error=type(e), name=module.__name__ - ), - "=" * 10, - format_exc(), - ] - ) - ) - - def _data_to_ast(self, data): - if isinstance(data, bytes): - data = self._loads(data.decode("utf-8")) - return ast.Module( - body=sum( - (cell_to_ast(object, transform=self.format).body for object in data["cells"]), [] - ) - ) - - def source_to_code(self, data, path): - return self._compile( - self.visit(self._data_to_ast(data)), path or "", "exec" - ) + exec_module = advanced_exec_module(NotebookLoader.exec_module) - from_filename = from_resource - -if __name__ == "__main__": - m = Notebook().from_filename("loader.ipynb") - -"""### Partial Loader -""" - -class Partial(Notebook): - """A partial import tool for notebooks. - - Sometimes notebooks don't work, but there may be useful code! - - with Partial(): - import Untitled as nb - assert nb.__exception__ - - if isinstance(nb.__exception__, AssertionError): - print("There was a False assertion.") - - Partial is useful in logging specific debugging approaches to the exception. - """ - __init__ = partialmethod(Notebook.__init__, exceptions=BaseException) - -"""### Lazy Loader - -The lazy loader is helpful for time consuming operations. The module is not evaluated until it is used the first time after loading. -""" - -class Lazy(Notebook): - """A lazy importer for notebooks. For long operations and a lot of data, the lazy importer delays imports until - an attribute is accessed the first time. - - with Lazy(): - import Untitled as nb - """ - __init__ = partialmethod(Notebook.__init__, lazy=True) - -"""# IPython Extensions -""" def load_ipython_extension(ip=None): add_path_hooks(Notebook, Notebook.EXTENSION_SUFFIXES) @@ -337,13 +289,6 @@ def load_ipython_extension(ip=None): def unload_ipython_extension(ip=None): remove_one_path_hook(Notebook) -def main(*files): - with ExitStack() as stack: - loader = stack.enter_context(Notebook("__main__")) - if not files: - files = sys.argv[1:] - for file in files: - loader.from_filename(file) """# Developer """ @@ -353,11 +298,14 @@ def main(*files): from utils.export import export except: from .utils.export import export - export("loader.ipynb", "../loader.py") + # export('loader.ipynb', '../loader.py') m = Notebook().from_filename("loader.ipynb") - __import__("doctest").testmod(m, verbose=2) + print(__import__("doctest").testmod(m)) -""" if __name__ == '__main__': - __import__('doctest').testmod(Notebook().from_filename('loader.ipynb'), verbose=2) +""" !jupyter nbconvert --to python --stdout loader.ipynb > ../loader.py """ +"""# More Information + +The `importnb.loader` module recreates basic Python importing abilities. Have a look at [`execute.ipynb`](execute.ipynb) for more advanced usages. +""" diff --git a/src/importnb/nbtest.py b/src/importnb/nbtest.py index 6113e9b..8e13388 100644 --- a/src/importnb/nbtest.py +++ b/src/importnb/nbtest.py @@ -17,6 +17,7 @@ __file__ = globals().get("__file__", "nbtest.ipynb") + def attach_doctest(module): """A function to include doctests in a unittest suite. """ @@ -28,6 +29,7 @@ def load_tests(loader, tests, ignore): module.load_tests = load_tests return module + def testmod( module, extras="", doctest=True, exit=True, verbosity=1, failfast=None, catchbreak=None ): @@ -47,20 +49,22 @@ def testmod( ... return module + class NotebookTest(Notebook): def exec_module(self, module): super().exec_module(module) testmod(module) + class _test(TestCase): def test_importnb_test(self): assert True + if __name__ == "__main__": export("nbtest.ipynb", "../nbtest.py") __import__("doctest").testmod(Notebook().from_filename("nbtest.ipynb")) m = NotebookTest().from_filename(__file__) testmod(m, "-f") - diff --git a/src/importnb/notebooks/capture.ipynb b/src/importnb/notebooks/capture.ipynb index 1ad5783..e7e9b00 100644 --- a/src/importnb/notebooks/capture.ipynb +++ b/src/importnb/notebooks/capture.ipynb @@ -14,16 +14,17 @@ "outputs": [], "source": [ "try:\n", - " from IPython.utils.capture import capture_output\n", + " from IPython.utils.capture import capture_output, CapturedIO\n", " from IPython import get_ipython\n", " assert get_ipython(), \"\"\"There is no interactive shell\"\"\"\n", "except:\n", " from contextlib import redirect_stdout, ExitStack\n", " from io import StringIO\n", + " import sys\n", " try:\n", " from contextlib import redirect_stderr\n", " except:\n", - " import sys\n", + " \n", " class redirect_stderr:\n", " def __init__(self, new_target):\n", " self._new_target = new_target\n", @@ -67,12 +68,18 @@ "\n", " @property\n", " def stderr(self): return self._stderr and self._stderr.getvalue() or ''\n", - "\n" + "\n", + " def show(self):\n", + " \"\"\"write my output to sys.stdout/err as appropriate\"\"\"\n", + " sys.stdout.write(self.stdout)\n", + " sys.stderr.write(self.stderr)\n", + " sys.stdout.flush()\n", + " sys.stderr.flush()" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": { "scrolled": false }, @@ -84,6 +91,13 @@ " export('capture.ipynb', '../capture.py')\n", " __import__('doctest').testmod()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/src/importnb/notebooks/decoder.ipynb b/src/importnb/notebooks/decoder.ipynb index 2efc098..572d55c 100644 --- a/src/importnb/notebooks/decoder.ipynb +++ b/src/importnb/notebooks/decoder.ipynb @@ -11,7 +11,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -38,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -61,13 +61,7 @@ " object['metadata'].update({\n", " 'lineno': len(s_and_end[0][:next].rsplit('\"source\":', 1)[0].splitlines())\n", " })\n", - " \n", - " if object['cell_type'] == 'markdown':\n", - " object['source'] = codify_markdown(object['source'])\n", - " object['outputs'] = []\n", - " object['cell_type'] = 'code'\n", - " object['execution_count'] = None\n", - " \n", + " \n", " for key in ('source', 'text'): \n", " if key in object: object[key] = ''.join(object[key])\n", " \n", @@ -76,90 +70,16 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ - " @singledispatch\n", - " def codify_markdown(string_or_list): raise TypeError(\"Markdown must be a string or a list.\")\n", - "\n", - " @codify_markdown.register(str)\n", - " def codify_markdown_string(str):\n", - " if '\"\"\"' in str: str = \"'''{}\\n'''\".format(str)\n", - " else: str = '\"\"\"{}\\n\"\"\"'.format(str)\n", - " return str\n", - "\n", - " @codify_markdown.register(list)\n", - " def codify_markdown_list(str):\n", - " return list(\n", - " map(\"{}\\n\".format, codify_markdown_string(\n", - " ''.join(str)\n", - " ).splitlines())\n", - " )\n", - "\n", - " \n", - " load = partial(_load, cls=LineNumberDecoder)\n", " loads = partial(_loads, cls=LineNumberDecoder)" ] }, { "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - " def cell_to_ast(object, transform=identity, prefix=False):\n", - " module = ast.increment_lineno(\n", - " ast.parse(\n", - " transform(\"\".join(object[\"source\"]))\n", - " ), object[\"metadata\"].get(\"lineno\", 1)\n", - " )\n", - " prefix and module.body.insert(0, ast.Expr(ast.Ellipsis())) \n", - " return module" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - " def transform_cells(object, transform=dedent):\n", - " for cell in object['cells']:\n", - " if 'source' in cell:\n", - " cell['source'] = transform(''.join(cell['source']))\n", - " return object\n", - " \n", - " def ast_from_cells(object, transform=identity):\n", - " import ast\n", - " module = ast.Module(body=[])\n", - " for cell in object['cells']:\n", - " module.body.extend(\n", - " ast.fix_missing_locations(\n", - " ast.increment_lineno(\n", - " ast.parse(''.join(cell['source'])), \n", - " cell['metadata'].get('lineno', 1)\n", - " )\n", - " ).body)\n", - " return module" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - " def loads_ast(object, loads=loads, transform=dedent, ast_transform=identity):\n", - " if isinstance(object, str):\n", - " object = loads(object)\n", - " object = transform_cells(object, transform)\n", - " return ast_from_cells(object, ast_transform)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, + "execution_count": 16, "metadata": { "scrolled": false }, diff --git a/src/importnb/notebooks/execute.ipynb b/src/importnb/notebooks/execute.ipynb index 6b2522b..de3bb9f 100644 --- a/src/importnb/notebooks/execute.ipynb +++ b/src/importnb/notebooks/execute.ipynb @@ -15,12 +15,12 @@ " \n", "An executed notebook contains a `__notebook__` attributes that is populated with cell outputs.\n", "\n", - " >>> assert nb.__notebook__\n", + " >>> assert nb._notebook\n", " \n", "The `__notebook__` attribute complies with `nbformat`\n", "\n", " >>> from nbformat.v4 import new_notebook\n", - " >>> assert new_notebook(**nb.__notebook__), \"\"\"The notebook is not a valid nbformat\"\"\"\n", + " >>> assert new_notebook(**nb._notebook), \"\"\"The notebook is not a valid nbformat\"\"\"\n", " " ] }, @@ -29,38 +29,96 @@ "execution_count": 1, "metadata": {}, "outputs": [], + "source": [ + " if globals().get('show', None):\n", + " print('I am tested.')" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], "source": [ " try:\n", - " \n", " from .capture import capture_output\n", - " from .loader import Notebook, lazy_loader_cls\n", - " from .decoder import loads_ast, identity, loads, dedent, cell_to_ast\n", + " from .loader import Notebook, advanced_exec_module\n", + " from .decoder import identity, loads, dedent\n", " except:\n", " from capture import capture_output\n", - " from loader import Notebook, lazy_loader_cls\n", - " from decoder import loads_ast, identity, loads, dedent, cell_to_ast\n", + " from loader import Notebook, advanced_exec_module\n", + " from decoder import identity, loads, dedent\n", "\n", " import inspect, sys, ast\n", " from functools import partialmethod, partial\n", " from importlib import reload, _bootstrap\n", - " from traceback import print_exc, format_exc\n", - " from warnings import warn\n", - " import traceback\n", + " from importlib._bootstrap import _call_with_frames_removed, _new_module\n", + "\n", + " import traceback \n", + " from traceback import print_exc, format_exc, format_tb\n", + " from pathlib import Path\n", + " \n", + " from ast import NodeTransformer, parse, Assign, literal_eval, dump, fix_missing_locations, Str, Tuple, Ellipsis, Interactive\n", + " from collections import ChainMap \n", + " \n", " __all__ = 'Notebook', 'Partial', 'reload', 'Lazy'" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Loaders that reproduce notebook outputs" + ] + }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - " from ast import NodeTransformer, parse, Assign, literal_eval, dump, fix_missing_locations, Str, Tuple, Ellipsis, Interactive" + " def loader_include_notebook(loader, module): \n", + " if not hasattr(module, '_notebook'):\n", + " module._notebook = loads(loader.get_data(loader.path).decode('utf-8'))" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + " class NotebookCells(Notebook):\n", + " \"\"\"The NotebookCells loader's contain a _notebook attributes containing a state of the notebook.\n", + " \n", + " >>> assert NotebookCells().from_filename('execute.ipynb', 'importnb.notebooks')._notebook\n", + " \"\"\"\n", + " @advanced_exec_module\n", + " def exec_module(self, module, **globals): \n", + " loader_include_notebook(self, module)\n", + " _call_with_frames_removed(exec, self.source_to_code(module._notebook), module.__dict__, module.__dict__)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + " if __name__ == '__main__':\n", + " nb = NotebookCells().from_filename('execute.ipynb', 'importnb.notebooks')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Recreating IPython output objectss" + ] + }, + { + "cell_type": "code", + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -68,156 +126,192 @@ " return {'name': name, 'output_type': 'stream', 'text': text}\n", "\n", " def new_error(Exception):\n", - " return {\n", - " 'ename': type(Exception).__name__, \n", - " 'output_type': 'error', \n", - " 'evalue': str(Exception),\n", - " 'traceback': traceback.format_tb(Exception.__traceback__)}\n", - " \n", + " return {'ename': type(Exception).__name__, \n", + " 'output_type': 'error', \n", + " 'evalue': str(Exception),\n", + " 'traceback': format_tb(Exception.__traceback__)}\n", "\n", " def new_display(object):\n", - " return {\n", - " 'data': object.data,\n", - " \"metadata\": {},\n", - " \"output_type\": \"display_data\" \n", - " }" + " return {'data': object.data,\n", + " \"metadata\": {},\n", + " \"output_type\": \"display_data\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Reproduce notebooks with the `Execute` class." ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ " class Execute(Notebook):\n", - " \"\"\"A SourceFileLoader for notebooks that provides line number debugginer in the JSON source.\"\"\"\n", - " def create_module(self, spec):\n", - " module = super().create_module(spec) \n", - " module.__notebook__ = self._loads(self.get_data(self.path).decode('utf-8')) \n", - " return module\n", + " \"\"\"The Execute loader reproduces outputs in the module._notebook attribute.\n", + " \n", + " >>> nb_raw = Notebook(display=True, stdout=True).from_filename('execute.ipynb', 'importnb.notebooks')\n", + " >>> with Execute(display=True, stdout=True) as loader:\n", + " ... nb = loader.from_filename('execute.ipynb', 'importnb.notebooks', show=True)\n", " \n", - " def _iter_cells(self, module):\n", - " for i, cell in enumerate(module.__notebook__['cells']):\n", - " if cell['cell_type'] == 'code':\n", - " yield self._compile(\n", - " fix_missing_locations(self.visit(cell_to_ast(\n", - " cell, transform=self.format, prefix=i > 0\n", - " ))), self.path or '', 'exec')\n", - " \n", - " \n", + " The loader includes the first markdown cell or leading block string as the docstring.\n", + " \n", + " >>> assert nb.__doc__ and nb_raw.__doc__ \n", + " >>> assert nb.__doc__ == nb_raw.__doc__\n", + " \n", + " Nothing should have been executed.\n", + " \n", + " >>> assert any(cell.get('outputs', None) for cell in nb._notebook['cells']) \n", + " \"\"\"\n", + " @advanced_exec_module \n", " def exec_module(self, module, **globals):\n", - " \"\"\"All exceptions specific in the context.\n", - " \"\"\"\n", - " module.__dict__.update(globals)\n", - " for cell in module.__notebook__['cells']:\n", + " # Remove the outputs\n", + " loader_include_notebook(self, module)\n", + " for cell in module._notebook['cells']: \n", " if 'outputs' in cell: cell['outputs'] = []\n", - " for i, code in enumerate(self._iter_cells(module)):\n", + " for i, cell in enumerate(module._notebook['cells']):\n", " error = None\n", - " with capture_output(stdout=self.stdout, stderr=self.stderr, display=self.display) as out:\n", + " with capture_output() as out:\n", " try: \n", - " _bootstrap._call_with_frames_removed(exec, code, module.__dict__, module.__dict__)\n", - " except BaseException as e: \n", - " error = new_error(e)\n", - " print(error)\n", + " _call_with_frames_removed(exec, self.source_to_code(cell, interactive=bool(i)), module.__dict__, module.__dict__)\n", + " except BaseException as Exception: \n", + " error = new_error(Exception)\n", " try:\n", - " module.__exception__ = e\n", - " raise e\n", - " except self._exceptions: ...\n", + " module.__exception__ = Exception\n", + " raise Exception\n", + " except self.exceptions: ...\n", " break\n", " finally:\n", - " if out.outputs: cell['outputs'] += [new_display(object) for object in out.outputs]\n", - " if out.stdout: \n", + " if 'outputs' in cell:\n", + " if out.outputs: cell['outputs'] += [new_display(object) for object in out.outputs]\n", + " if out.stdout: cell['outputs'] += [new_stream(out.stdout)]\n", + " if error: cell['outputs'] += [error]\n", + " if out.stderr: cell['outputs'] += [new_stream(out.stderr, 'stderr')]\n", + " out.show()\n", "\n", - " cell['outputs'] += [new_stream(out.stdout)]\n", - " if error: cell['outputs'] += [error]\n", - " if out.stderr: cell['outputs'] += [new_stream(out.stderr, 'stderr')]\n", - " " + " def source_to_code(loader, object, path=None, interactive=True):\n", + " \"\"\"Transform ast modules into Interactive and Expression nodes. This \n", + " will allow the cell outputs to be captured. `interactive` is only true for\n", + " the first markdown cell.\n", + " \"\"\"\n", + " \n", + " node = loader.visit(object)\n", + " \n", + " if isinstance(node, ast.Expr):\n", + " \"\"\"An expression is a special case where a markdown cell was \n", + " turned into a docstring\n", + " \"\"\"\n", + " if interactive:\n", + " \"\"\"An expression will not print to the stdout.\"\"\"\n", + " node = ast.Expression(body=node.value)\n", + " mode = 'eval'\n", + " else: \n", + " \"\"\"This tree replaces the docstring\"\"\"\n", + " node = ast.Module(body=[node])\n", + " mode = 'exec'\n", + " elif isinstance(node, ast.Module):\n", + " \"\"\"In the exectuion mode we use interactive nodes to capture stdouts.\"\"\"\n", + " node = ast.Interactive(body=node.body)\n", + " mode = 'single'\n", + " return compile(node, path or \"\", mode) " ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 8, "metadata": {}, + "outputs": [], "source": [ " if __name__ == '__main__':\n", - " m = Execute(stdout=True).from_filename('loader.ipynb')" + " nb = Execute().from_filename('execute.ipynb', 'importnb.notebooks')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Parameterizing notebooks" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ - " class ParameterizeNode(NodeTransformer):\n", - " visit_Module = NodeTransformer.generic_visit\n", + " class AssignmentFinder(NodeTransformer):\n", + " visit_Interactive = visit_Module = NodeTransformer.generic_visit\n", " \n", - " def visit_Assign(FreeStatement, node):\n", + " def visit_Assign(self, node):\n", " if len(node.targets):\n", " try:\n", " if not getattr(node.targets[0], 'id', '_').startswith('_'):\n", " literal_eval(node.value)\n", " return node\n", - " except: assert True, \"\"\"The target can not will not literally evaluate.\"\"\"\n", - " return None\n", - " \n", + " except: ...\n", + " \n", " def generic_visit(self, node): ..." ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ - " class ExecuteNode(ParameterizeNode):\n", + " class AssignmentIgnore(AssignmentFinder):\n", " def visit_Assign(self, node):\n", - " if super().visit_Assign(node): return ast.Expr(Ellipsis())\n", - " return node \n", + " if isinstance(super().visit_Assign(node), ast.Assign):\n", + " return ast.Expr(ast.NameConstant(value=None))\n", + " return node\n", " \n", - " def generic_visit(self, node): return node" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - " def vars_to_sig(**vars):\n", - " \"\"\"Create a signature for a dictionary of names.\"\"\"\n", - " from inspect import Parameter, Signature\n", - " return Signature([Parameter(str, Parameter.KEYWORD_ONLY, default = vars[str]) for str in vars])\n" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [], - "source": [ - " from collections import ChainMap" + " generic_visit = NodeTransformer.generic_visit" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ - " class Parameterize(Execute, ExecuteNode):\n", + " class Parameterize(Execute):\n", + " \"\"\"Discover any literal ast expression and create parameters from them. \n", + " \n", + " >>> f = Parameterize().from_filename('execute.ipynb', 'importnb.notebooks')\n", + " >>> assert 'a_variable_to_parameterize' in f.__signature__.parameters\n", + " >>> assert f(a_variable_to_parameterize=100).a_variable_to_parameterize == 100\n", + " \n", + " Parametize is a NodeTransformer that import any nodes return by Parameterize Node.\n", + " \n", + " >>> assert len(Parameterize().visit(ast.parse('''\n", + " ... foo = 42\n", + " ... bar = foo''')).body) ==2\n", + " \"\"\" \n", " def create_module(self, spec):\n", " module = super().create_module(spec)\n", - " nodes = self._data_to_ast(module.__notebook__)\n", + " \n", + " # Import the notebook when parameterize is imported\n", + " loader_include_notebook(self, module)\n", + " \n", + " node = Notebook().visit(module._notebook)\n", + " \n", + " # Extra effort to supply a docstring\n", " doc = None\n", - " if isinstance(nodes, ast.Module) and nodes.body: \n", - " node = nodes.body[0]\n", - " if isinstance(node, ast.Expr) and isinstance(node.value, ast.Str):\n", - " doc = node\n", - " params = ParameterizeNode().visit(nodes)\n", + " if isinstance(node, ast.Module) and node.body: \n", + " _node = node.body[0]\n", + " if isinstance(_node, ast.Expr) and isinstance(_node.value, ast.Str):\n", + " doc = _node\n", + " \n", + " # Discover the parameterizable nodes\n", + " params = AssignmentFinder().visit(node)\n", + " # Include the string in the compilation\n", " doc and params.body.insert(0, doc)\n", - " exec(compile(\n", - " params, '', 'exec'\n", - " ), module.__dict__, module.__dict__)\n", + " \n", + " # Supply the literal parameter values as module globals.\n", + " exec(compile(params, '', 'exec'), module.__dict__, module.__dict__)\n", " return module\n", " \n", " def from_filename(self, filename, path=None, **globals):\n", @@ -230,15 +324,50 @@ " \n", " recall.__signature__ = vars_to_sig(**{k: v for k, v in module.__dict__.items() if not k.startswith('_')})\n", " recall.__doc__ = module.__doc__\n", - " return recall" + " return recall\n", + " \n", + " def visit(self, node):\n", + " return AssignmentIgnore().visit(super().visit(node))" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + " def vars_to_sig(**vars):\n", + " \"\"\"Create a signature for a dictionary of names.\"\"\"\n", + " from inspect import Parameter, Signature\n", + " return Signature([Parameter(str, Parameter.KEYWORD_ONLY, default = vars[str]) for str in vars])" + ] + }, + { + "cell_type": "code", + "execution_count": 13, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1000\n" + ] + } + ], "source": [ " if __name__ == '__main__':\n", - " f = Parameterize().from_filename('execute.ipynb')" + " f = Parameterize(display=True).from_filename('execute.ipynb', 'importnb.notebooks')\n", + " print(f(a_variable_to_parameterize=1000).a_variable_to_parameterize)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + " a_variable_to_parameterize = 42" ] }, { @@ -250,11 +379,109 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": { "scrolled": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Trying:\n", + " import importnb \n", + "Expecting nothing\n", + "ok\n", + "Trying:\n", + " from importnb import notebooks\n", + "Expecting nothing\n", + "ok\n", + "Trying:\n", + " with Execute(stdout=True):\n", + " from importnb.notebooks import execute as nb\n", + "Expecting nothing\n", + "ok\n", + "Trying:\n", + " assert nb._notebook\n", + "Expecting nothing\n", + "ok\n", + "Trying:\n", + " from nbformat.v4 import new_notebook\n", + "Expecting nothing\n", + "ok\n", + "Trying:\n", + " assert new_notebook(**nb._notebook), \"\"\"The notebook is not a valid nbformat\"\"\"\n", + "Expecting nothing\n", + "ok\n", + "Trying:\n", + " nb_raw = Notebook(display=True, stdout=True).from_filename('execute.ipynb', 'importnb.notebooks')\n", + "Expecting nothing\n", + "ok\n", + "Trying:\n", + " with Execute(display=True, stdout=True) as loader:\n", + " nb = loader.from_filename('execute.ipynb', 'importnb.notebooks', show=True)\n", + "Expecting nothing\n", + "ok\n", + "Trying:\n", + " assert nb.__doc__ and nb_raw.__doc__ \n", + "Expecting nothing\n", + "ok\n", + "Trying:\n", + " assert nb.__doc__ == nb_raw.__doc__\n", + "Expecting nothing\n", + "ok\n", + "Trying:\n", + " assert any(cell.get('outputs', None) for cell in nb._notebook['cells']) \n", + "Expecting nothing\n", + "ok\n", + "Trying:\n", + " assert NotebookCells().from_filename('execute.ipynb', 'importnb.notebooks')._notebook\n", + "Expecting nothing\n", + "ok\n", + "Trying:\n", + " f = Parameterize().from_filename('execute.ipynb', 'importnb.notebooks')\n", + "Expecting nothing\n", + "ok\n", + "Trying:\n", + " assert 'a_variable_to_parameterize' in f.__signature__.parameters\n", + "Expecting nothing\n", + "ok\n", + "Trying:\n", + " assert f(a_variable_to_parameterize=100).a_variable_to_parameterize == 100\n", + "Expecting nothing\n", + "ok\n", + "Trying:\n", + " assert len(Parameterize().visit(ast.parse('''\n", + " foo = 42\n", + " bar = foo''')).body) ==2\n", + "Expecting nothing\n", + "ok\n", + "14 items had no tests:\n", + " execute.AssignmentFinder\n", + " execute.AssignmentFinder.generic_visit\n", + " execute.AssignmentFinder.visit_Assign\n", + " execute.AssignmentIgnore\n", + " execute.AssignmentIgnore.visit_Assign\n", + " execute.Execute.source_to_code\n", + " execute.Parameterize.create_module\n", + " execute.Parameterize.from_filename\n", + " execute.Parameterize.visit\n", + " execute.loader_include_notebook\n", + " execute.new_display\n", + " execute.new_error\n", + " execute.new_stream\n", + " execute.vars_to_sig\n", + "4 items passed all tests:\n", + " 6 tests in execute\n", + " 5 tests in execute.Execute\n", + " 1 tests in execute.NotebookCells\n", + " 4 tests in execute.Parameterize\n", + "16 tests in 18 items.\n", + "16 passed and 0 failed.\n", + "Test passed.\n" + ] + } + ], "source": [ " if __name__ == '__main__':\n", " try: from utils.export import export\n", @@ -268,7 +495,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "For more information check out [`importnb`](https://github.com/deathbeds/importnb)" + "How do the interactive nodes work?" ] }, { diff --git a/src/importnb/notebooks/loader.ipynb b/src/importnb/notebooks/loader.ipynb index fd8bc99..b15c60e 100644 --- a/src/importnb/notebooks/loader.ipynb +++ b/src/importnb/notebooks/loader.ipynb @@ -14,15 +14,15 @@ " \n", "### `importnb.Partial` \n", "\n", - " >>> with Partial(): \n", + " >>> with Notebook(exceptions=BaseException): \n", " ... from importnb.notebooks import loader\n", - " >>> assert loader.__exception__ is None\n", + " >>> assert loader._exception is None\n", " \n", "## There is a [lazy importer]()\n", "\n", "The Lazy importer will delay the module execution until it is used the first time. It is a useful approach for delaying visualization or data loading.\n", "\n", - " >>> with Lazy(): \n", + " >>> with Notebook(lazy=True): \n", " ... from importnb.notebooks import loader\n", " \n", "## Loading from resources\n", @@ -39,32 +39,37 @@ "\n", " >>> with Notebook(stdout=True, stderr=True, display=True, globals=dict(show=True)):\n", " ... from importnb.notebooks import loader\n", - " >>> assert loader.__output__\n", - " \n", - " # loader.__output__.stdout" + " >>> assert loader._capture\n", + "\n", + "## Assigning globals\n", + "\n", + " >>> nb = Notebook(stdout=True, globals={'show': True}).from_filename('loader.ipynb', 'importnb.notebooks')\n", + " >>> assert nb._capture.stdout" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - " if 'show' in globals(): print(\"Catch me if you can\")" + " if globals().get('show', None): print(\"Catch me if you can\")" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ " try:\n", - " from .capture import capture_output\n", - " from .decoder import identity, loads, dedent, cell_to_ast\n", + " from .capture import capture_output, CapturedIO\n", + " from .decoder import identity, loads, dedent\n", + " from .path_hooks import PathHooksContext, modify_sys_path, add_path_hooks, remove_one_path_hook\n", " except:\n", - " from capture import capture_output\n", - " from decoder import identity, loads, dedent, cell_to_ast\n", + " from capture import capture_output, CapturedIO\n", + " from decoder import identity, loads, dedent\n", + " from path_hooks import PathHooksContext, modify_sys_path, add_path_hooks, remove_one_path_hook\n", "\n", " import inspect, sys, ast\n", " from copy import copy\n", @@ -88,142 +93,51 @@ " return _SpecMethods(spec).init_module_attrs(module)\n", " \n", " from io import StringIO\n", - " from functools import partialmethod, partial\n", + " from functools import partialmethod, partial, wraps, singledispatch\n", " from importlib import reload\n", - " from traceback import print_exc, format_exc\n", - " from warnings import warn\n", + " from traceback import print_exc, format_exc, format_tb\n", " from contextlib import contextmanager, ExitStack\n", " from pathlib import Path\n", " try:\n", " from importlib.resources import path\n", " except:\n", " from importlib_resources import path\n", + " \n", + " from ast import NodeTransformer, parse, Assign, literal_eval, dump, fix_missing_locations, Str, Tuple, Ellipsis, Interactive\n", + " from collections import ChainMap \n", " \n", " __all__ = 'Notebook', 'Partial', 'reload', 'Lazy'" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## `sys.path_hook` modifiers" - ] - }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - " @contextmanager\n", - " def modify_file_finder_details():\n", - " \"\"\"yield the FileFinder in the sys.path_hooks that loads Python files and assure\n", - " the import cache is cleared afterwards. \n", - " \n", - " Everything goes to shit if the import cache is not cleared.\"\"\"\n", - " \n", - " for id, hook in enumerate(sys.path_hooks):\n", - " try:\n", - " closure = inspect.getclosurevars(hook).nonlocals\n", - " except TypeError: continue\n", - " if issubclass(closure['cls'], FileFinder):\n", - " sys.path_hooks.pop(id)\n", - " details = list(closure['loader_details'])\n", - " yield details\n", - " break\n", - " sys.path_hooks.insert(id, FileFinder.path_hook(*details))\n", - " sys.path_importer_cache.clear()" + " class ImportNbException(BaseException):\n", + " \"\"\"ImportNbException allows all exceptions to be raised, a null except statement always passes.\"\"\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Update the file_finder details with functions to append and remove the [loader details](https://docs.python.org/3.7/library/importlib.html#importlib.machinery.FileFinder)." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - " def add_path_hooks(loader: SourceFileLoader, extensions, *, position=0):\n", - " \"\"\"Update the FileFinder loader in sys.path_hooks to accomodate a {loader} with the {extensions}\"\"\"\n", - " with modify_file_finder_details() as details:\n", - " details.insert(position, (loader, extensions))\n", + "## Converting cells to code\n", "\n", - " def remove_one_path_hook(loader):\n", - " loader = lazy_loader_cls(loader)\n", - " with modify_file_finder_details() as details:\n", - " _details = list(details)\n", - " for ct, (cls, ext) in enumerate(_details):\n", - " cls = lazy_loader_cls(cls)\n", - " if cls == loader:\n", - " details.pop(ct)\n", - " break" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - " def lazy_loader_cls(loader):\n", - " \"\"\"Extract the loader contents of a lazy loader in the import path.\"\"\"\n", - " try:\n", - " return inspect.getclosurevars(loader).nonlocals.get('cls', loader)\n", - " except:\n", - " return loader" + "These functions are attached to the loaders.s" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Loader Context Manager\n", - "\n", - "`importnb` uses a context manager to assure that the traditional import system behaviors as expected. If the loader is permenantly available then it may create some unexpected import behaviors." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - " class ImportNbException(BaseException):\n", - " \"\"\"ImportNbException allows all exceptions to be raised, a null except statement always passes.\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - " class PathHooksContext:\n", - " def __enter__(self, position=0): \n", - " add_path_hooks(self.prepare(self), self.EXTENSION_SUFFIXES, position=position)\n", - " return self\n", - " \n", - " def __exit__(self, *excepts): remove_one_path_hook(self)\n", - "\n", - " def prepare(self, loader):\n", - " if self._lazy: \n", - " try:\n", - " from importlib.util import LazyLoader\n", - " if self._lazy: \n", - " loader = LazyLoader.factory(loader)\n", - " except:\n", - " ImportWarning(\"\"\"LazyLoading is only available in > Python 3.5\"\"\")\n", - " return loader\n" + "## Loading from resources" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -234,7 +148,7 @@ "\n", " This still needs some work for packages.\n", " \n", - " >>> assert from_resource(Notebook(), 'loader.ipynb', 'importnb.notebooks')\n", + " >> assert from_resource(Notebook(), 'loader.ipynb', 'importnb.notebooks')\n", " \"\"\"\n", " with ExitStack() as stack:\n", " if resource is not None:\n", @@ -263,192 +177,161 @@ ] }, { - "cell_type": "code", - "execution_count": 9, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - " @contextmanager\n", - " def modify_sys_path(file):\n", - " \"\"\"This is only invoked when using from_resource.\"\"\"\n", - " path = str(Path(file).parent)\n", - " if path not in map(str, map(Path, sys.path)):\n", - " yield sys.path.insert(0, path)\n", - " sys.path = [object for object in sys.path if str(Path(object)) != path]\n", - " else: yield" + "# The Notebook Loader" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ - " class Notebook(SourceFileLoader, PathHooksContext, capture_output, ast.NodeTransformer):\n", - " \"\"\"A SourceFileLoader for notebooks that provides line number debugginer in the JSON source.\"\"\"\n", + " class NotebookLoader(SourceFileLoader, PathHooksContext, NodeTransformer):\n", + " \"\"\"The simplest implementation of a Notebook Source File Loader.\n", + " >>> with NotebookLoader():\n", + " ... from importnb.notebooks import decoder\n", + " >>> assert decoder.__file__.endswith('.ipynb')\n", + " \"\"\"\n", " EXTENSION_SUFFIXES = '.ipynb',\n", - " \n", - " _compile = staticmethod(compile)\n", - " _loads = staticmethod(loads)\n", - " format = _transform = staticmethod(dedent)\n", - " \n", - " __slots__ = 'stdout', 'stderr', 'display', '_lazy', '_exceptions', 'globals'\n", + " __slots__ = 'name', 'path',\n", " \n", " def __init__(\n", - " self, fullname=None, path=None, *, \n", - " stdout=False, stderr=False, display=False,\n", - " lazy=False, exceptions=ImportNbException, globals=None\n", - " ): \n", - " SourceFileLoader.__init__(self, fullname, path)\n", - " capture_output.__init__(self, stdout=stdout, stderr=stderr, display=display)\n", - " self._lazy = lazy\n", - " self._exceptions = exceptions\n", - " self.globals = {} if globals is None else globals\n", - " \n", + " self, fullname=None, path=None\n", + " ): super().__init__(fullname, path)\n", + "\n", + " format = staticmethod(dedent)\n", + " from_filename = from_resource\n", + " \n", " def __call__(self, fullname=None, path=None): \n", " self= copy(self)\n", " return SourceFileLoader.__init__(self, str(fullname), str(path)) or self\n", + "\n", + " def source_to_code(loader, object, path=None):\n", + " node = loader.visit(object)\n", + " return compile(node, path or \"\", 'exec')\n", " \n", - " def visit(self, node): \n", - " node = super().visit(node)\n", - " return ast.fix_missing_locations(super().visit(node))\n", - " \n", - " def create_module(self, spec):\n", - " module = _new_module(spec.name)\n", - " _init_module_attrs(spec, module)\n", - " module.__exception__ = None\n", - " module.__dict__.update(self.globals)\n", - " return module\n", - " \n", - " def exec_module(self, module, **globals):\n", - " \"\"\"All exceptions specific in the context.\n", - " \"\"\"\n", - " module.__dict__.update(globals)\n", - " with capture_output(stdout=self.stdout, stderr=self.stderr, display=self.display) as out:\n", - " module.__output__ = out\n", - " try: \n", - " super().exec_module(module)\n", - " except self._exceptions as e:\n", - " \"\"\"Display a message if an error is escaped.\"\"\"\n", - " module.__exception__ = e\n", - " warn('.'.join([\n", - " \"\"\"{name} was partially imported with a {error}\"\"\".format(\n", - " error = type(e), name=module.__name__\n", - " ), \"=\"*10, format_exc()]))\n", - " \n", - " def _data_to_ast(self, data):\n", - " if isinstance(data, bytes):\n", - " data = self._loads(data.decode('utf-8'))\n", - " return ast.Module(body=sum((\n", - " cell_to_ast(object, transform=self.format).body \n", - " for object in data['cells']), []))\n", - " \n", - " def source_to_code(self, data, path):\n", - " return self._compile(\n", - " self.visit(self._data_to_ast(data)), path or '', 'exec')\n", - " \n", - " from_filename = from_resource" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - " if __name__ == '__main__':\n", - " m = Notebook().from_filename('loader.ipynb')" + " def visit(self, node, **opts): \n", + " if isinstance(node, bytes):node = loads(node.decode('utf-8'))\n", + "\n", + " if isinstance(node, dict):\n", + " if 'cells' in node: \n", + " body = []\n", + " for cell in node['cells']:\n", + " _node = self.visit(cell)\n", + " _node = ast.increment_lineno(\n", + " _node, cell[\"metadata\"].get(\"lineno\", 1))\n", + " body.extend(getattr(_node, 'body', [_node]))\n", + " node = ast.Module(body=body)\n", + "\n", + " elif 'source' in node:\n", + " source = \"\".join(node[\"source\"])\n", + " if node['cell_type'] == 'markdown':\n", + " node = ast.Expr(ast.Str(s=source))\n", + " elif node['cell_type'] == 'code':\n", + " node = ast.parse(\n", + " self.format(source), self.path or '', 'exec')\n", + " else: \n", + " node = ast.Module(body=[])\n", + " return ast.fix_missing_locations(super().visit(node)) \n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Partial Loader" + "## An advanced `exec_module` decorator." ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ - " class Partial(Notebook): \n", - " \"\"\"A partial import tool for notebooks.\n", + " def advanced_exec_module(exec_module):\n", + " \"\"\"Decorate `SourceFileLoader.exec_module` objects with abilities to:\n", + " * Capture output in Python and IPython\n", + " * Prepopulate a model namespace.\n", + " * Allow exceptions while notebooks are loading.s\n", " \n", - " Sometimes notebooks don't work, but there may be useful code!\n", - " \n", - " with Partial():\n", - " import Untitled as nb\n", - " assert nb.__exception__\n", - " \n", - " if isinstance(nb.__exception__, AssertionError):\n", - " print(\"There was a False assertion.\")\n", - " \n", - " Partial is useful in logging specific debugging approaches to the exception.\n", + " >>> assert advanced_exec_module(SourceFileLoader.exec_module)\n", " \"\"\"\n", - " __init__ = partialmethod(Notebook.__init__, exceptions=BaseException)" + " def _exec_module(loader, module, **globals):\n", + " module._exception = None\n", + " module.__dict__.update(globals) \n", + " with capture_output(stdout=loader.stdout, stderr=loader.stderr, display=loader.display) as out:\n", + " module._capture = out\n", + " try: exec_module(loader, module)\n", + " except loader.exceptions as Exception: \n", + " module._exception = Exception\n", + " return _exec_module\n", + "\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Lazy Loader\n", - "\n", - "The lazy loader is helpful for time consuming operations. The module is not evaluated until it is used the first time after loading." + "# The Advanced Notebook loader" ] }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ - " class Lazy(Notebook): \n", - " \"\"\"A lazy importer for notebooks. For long operations and a lot of data, the lazy importer delays imports until \n", - " an attribute is accessed the first time.\n", + " class Notebook(NotebookLoader, capture_output):\n", + " \"\"\"The Notebook loader is an advanced loader for IPython notebooks:\n", + " \n", + " * Capture stdout, stderr, and display objects.\n", + " * Partially evaluate notebook with known exceptions.\n", + " * Supply extra global values into notebooks.\n", " \n", - " with Lazy():\n", - " import Untitled as nb\n", + " >>> assert Notebook().from_filename('loader.ipynb', 'importnb.notebooks')\n", " \"\"\"\n", - " __init__ = partialmethod(Notebook.__init__, lazy=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# IPython Extensions" + " EXTENSION_SUFFIXES = '.ipynb',\n", + " \n", + " format = _transform = staticmethod(dedent)\n", + " \n", + " __slots__ = 'stdout', 'stderr', 'display', '_lazy', 'exceptions', 'globals'\n", + " \n", + " def __init__(\n", + " self, fullname=None, path=None, *, \n", + " stdout=False, stderr=False, display=False,\n", + " lazy=False, globals=None, exceptions=ImportNbException\n", + " ): \n", + " super().__init__(fullname, path)\n", + " capture_output.__init__(self, stdout=stdout, stderr=stderr, display=display)\n", + " self._lazy = lazy\n", + " self.globals = {} if globals is None else globals\n", + " self.exceptions = exceptions\n", + " \n", + " def create_module(self, spec):\n", + " module = _new_module(spec.name)\n", + " _init_module_attrs(spec, module)\n", + " module.__dict__.update(self.globals)\n", + " return module\n", + " \n", + " exec_module = advanced_exec_module(NotebookLoader.exec_module)\n" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ - " def load_ipython_extension(ip=None): \n", + " def load_ipython_extension(ip=None):\n", " add_path_hooks(Notebook, Notebook.EXTENSION_SUFFIXES)\n", - " def unload_ipython_extension(ip=None): \n", + "\n", + " def unload_ipython_extension(ip=None):\n", " remove_one_path_hook(Notebook)" ] }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [], - "source": [ - " def main(*files):\n", - " with ExitStack() as stack:\n", - " loader = stack.enter_context(Notebook('__main__'))\n", - " if not files:\n", - " files = sys.argv[1:]\n", - " for file in files:\n", - " loader.from_filename(file)" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -458,92 +341,14 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 38, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Trying:\n", - " m = Notebook().from_filename('loader.ipynb', 'importnb.notebooks')\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert m and m.Notebook\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " with Partial(): \n", - " from importnb.notebooks import loader\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert loader.__exception__ is None\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " with Lazy(): \n", - " from importnb.notebooks import loader\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " from importnb.loader import from_resource\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert from_resource(Notebook(), 'loader.ipynb', 'importnb.notebooks')\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert Notebook().from_filename('loader.ipynb', 'importnb.notebooks')\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert Notebook().from_filename(m.__file__)\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " with Notebook(stdout=True, stderr=True, display=True, globals=dict(show=True)):\n", - " from importnb.notebooks import loader\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert loader.__output__\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert from_resource(Notebook(), 'loader.ipynb', 'importnb.notebooks')\n", - "Expecting nothing\n", - "ok\n", - "21 items had no tests:\n", - " loader.ImportNbException\n", - " loader.Lazy\n", - " loader.Notebook\n", - " loader.Notebook.__call__\n", - " loader.Notebook.__init__\n", - " loader.Notebook._data_to_ast\n", - " loader.Notebook.create_module\n", - " loader.Notebook.exec_module\n", - " loader.Notebook.source_to_code\n", - " loader.Notebook.visit\n", - " loader.Partial\n", - " loader.PathHooksContext\n", - " loader.PathHooksContext.__enter__\n", - " loader.PathHooksContext.__exit__\n", - " loader.PathHooksContext.prepare\n", - " loader.add_path_hooks\n", - " loader.lazy_loader_cls\n", - " loader.load_ipython_extension\n", - " loader.main\n", - " loader.remove_one_path_hook\n", - " loader.unload_ipython_extension\n", - "2 items passed all tests:\n", - " 11 tests in loader\n", - " 1 tests in loader.from_resource\n", - "12 tests in 23 items.\n", - "12 passed and 0 failed.\n", - "Test passed.\n" + "TestResults(failed=0, attempted=17)\n" ] } ], @@ -551,19 +356,34 @@ " if __name__ == '__main__':\n", " try: from utils.export import export\n", " except: from .utils.export import export\n", - " export('loader.ipynb', '../loader.py')\n", + " # export('loader.ipynb', '../loader.py')\n", " m = Notebook().from_filename('loader.ipynb')\n", - " __import__('doctest').testmod(m, verbose=2)" + " print(__import__('doctest').testmod(m))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - " if __name__ == '__main__':\n", - " __import__('doctest').testmod(Notebook().from_filename('loader.ipynb'), verbose=2)" + " !jupyter nbconvert --to python --stdout loader.ipynb > ../loader.py" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# More Information\n", + "\n", + "The `importnb.loader` module recreates basic Python importing abilities. Have a look at [`execute.ipynb`](execute.ipynb) for more advanced usages." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, diff --git a/src/importnb/notebooks/parameterize.ipynb b/src/importnb/notebooks/parameterize.ipynb deleted file mode 100644 index 1564548..0000000 --- a/src/importnb/notebooks/parameterize.ipynb +++ /dev/null @@ -1,321 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - " try:\n", - " from .loader import Notebook\n", - " from .decoder import loads_ast\n", - " except:\n", - " from loader import Notebook\n", - " from decoder import loads_ast\n", - " from inspect import getsource\n", - " \n", - " from types import ModuleType" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Are single target `ast.Expr` that will `ast.literal_eval` is a possible parameter." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - " from ast import NodeTransformer, parse, Assign, literal_eval, dump, fix_missing_locations, Str, Tuple\n", - " class FreeStatementFinder(NodeTransformer):\n", - " def __init__(self, params=None, globals=None):\n", - " self.params = params if params is not None else []\n", - " self.globals = globals if globals is not None else {}\n", - " \n", - " visit_Module = NodeTransformer.generic_visit\n", - " \n", - " def visit_Assign(FreeStatement, node):\n", - " if len(node.targets):\n", - " try:\n", - " if not getattr(node.targets[0], 'id', '_').startswith('_'):\n", - " FreeStatement.globals[node.targets[0].id] = literal_eval(node.value)\n", - " return \n", - " except: assert True, \"\"\"The target can not will not literally evaluate.\"\"\"\n", - " return node\n", - " \n", - " def generic_visit(self, node): return node\n", - " \n", - " def __call__(FreeStatement, nodes): return FreeStatement.globals, fix_missing_locations(FreeStatement.visit(nodes))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# `Parameterize` notebooks\n", - "\n", - "`Parameterize` is callable version of a notebook. It uses `pidgin` to load the `NotebookNode` and evaluates the `FreeStatement`s to discover the signature." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - " class Parameterize:\n", - " \"\"\"Parameterize takes a module, filename, or notebook dictionary and returns callable object that parameterizes the notebook module.\n", - " \n", - " f = Parameterize('parameterize.ipynb')\n", - " \"\"\"\n", - " def __init__(\n", - " self, object=None\n", - " ):\n", - " from importnb.capture import capture_output\n", - " from pathlib import Path\n", - " from json import load, loads\n", - " self.object = object\n", - "\n", - " self.__file__ = None\n", - " \n", - " if isinstance(object, ModuleType):\n", - " self.__file__ = object.__file__\n", - " object = getsource(object)\n", - " elif isinstance(object, (Path, str)):\n", - " self.__file__ = object\n", - " with open(str(object)) as f: \n", - " object = f.read()\n", - " elif isinstance(object, dict): ...\n", - " else: raise ValueError(\"object must be a module, file string, or dict.\")\n", - " \n", - " self.__variables__, self.__ast__ = \\\n", - " FreeStatementFinder()(loads_ast(object))\n", - " self.__signature__ = self.vars_to_sig(**self.__variables__)\n", - "\n", - " def __call__(self, **dict):\n", - " self = __import__('copy').copy(self)\n", - " self.__dict__.update(self.__variables__)\n", - " self.__dict__.update(dict)\n", - " exec(compile(self.__ast__, self.__file__ or '', 'exec'), *[self.__dict__]*2)\n", - " return self\n", - " \n", - " def interact(Parameterize): \n", - " \"\"\"Use the ipywidgets.interact to explore the parameterized notebook.\"\"\"\n", - " return __import__('ipywidgets').interact(Parameterize)\n", - " \n", - " @staticmethod\n", - " def vars_to_sig(**vars):\n", - " \"\"\"Create a signature for a dictionary of names.\"\"\"\n", - " from inspect import Parameter, Signature\n", - " return Signature([Parameter(str, Parameter.KEYWORD_ONLY, default = vars[str]) for str in vars])\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Examples that do work" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - " import sys\n", - " \n", - " param = 'xyz'\n", - " extraparam = 42" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Examples that do *not* work" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - " \"\"\"Parameters are not created when literal_eval fails.\"\"\"\n", - " noparam0 = Parameterize\n", - " \n", - " \"\"\"Multiple target assignments are ignored.\"\"\"\n", - " noparam1, noparam2 = 'xyz', 42" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Developer" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Trying:\n", - " default = f()\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert default.param == default.noparam1 == 'xyz' and default.noparam2 == 42\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert all(str not in default.__signature__.parameters for str in ('noparam', 'noparam1', 'noparam2'))\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert callable(f)\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " new = f(param=10)\n", - "Expecting nothing\n", - "ok\n", - "Trying:\n", - " assert new.param is 10 and new.extraparam is 42\n", - "Expecting nothing\n", - "ok\n", - "11 items had no tests:\n", - " __main__\n", - " __main__.FreeStatementFinder\n", - " __main__.FreeStatementFinder.__call__\n", - " __main__.FreeStatementFinder.__init__\n", - " __main__.FreeStatementFinder.generic_visit\n", - " __main__.FreeStatementFinder.visit_Assign\n", - " __main__.Parameterize\n", - " __main__.Parameterize.__call__\n", - " __main__.Parameterize.__init__\n", - " __main__.Parameterize.interact\n", - " __main__.Parameterize.vars_to_sig\n", - "3 items passed all tests:\n", - " 3 tests in __main__.__test__.default\n", - " 1 tests in __main__.__test__.imports\n", - " 2 tests in __main__.__test__.reuse\n", - "6 tests in 14 items.\n", - "6 passed and 0 failed.\n", - "Test passed.\n" - ] - } - ], - "source": [ - " __test__ = dict(\n", - " imports=\"\"\"\n", - " >>> assert callable(f)\n", - " \"\"\",\n", - " default=\"\"\"\n", - " >>> default = f()\n", - " >>> assert default.param == default.noparam1 == 'xyz' and default.noparam2 == 42\n", - " >>> assert all(str not in default.__signature__.parameters for str in ('noparam', 'noparam1', 'noparam2'))\n", - " \"\"\",\n", - " reuse=\"\"\"\n", - " >>> new = f(param=10)\n", - " >>> assert new.param is 10 and new.extraparam is 42\"\"\",\n", - " )\n", - " if __name__ == '__main__':\n", - " f = Parameterize(globals().get('__file__', 'parameterize.ipynb'))\n", - " __import__('doctest').testmod(verbose=1)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "scrolled": false - }, - "outputs": [], - "source": [ - " if __name__ == '__main__':\n", - " # export('parameterize.ipynb', '../parameterize.py')\n", - " __import__('doctest').testmod()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "p6", - "language": "python", - "name": "other-env" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.3" - }, - "toc": { - "colors": { - "hover_highlight": "#DAA520", - "running_highlight": "#FF0000", - "selected_highlight": "#FFD700" - }, - "moveMenuLeft": true, - "nav_menu": { - "height": "30px", - "width": "252px" - }, - "navigate_menu": true, - "number_sections": true, - "sideBar": true, - "threshold": 4, - "toc_cell": false, - "toc_section_display": "block", - "toc_window_display": false, - "widenNotebook": false - }, - "varInspector": { - "cols": { - "lenName": 16, - "lenType": 16, - "lenVar": 40 - }, - "kernels_config": { - "python": { - "delete_cmd_postfix": "", - "delete_cmd_prefix": "del ", - "library": "var_list.py", - "varRefreshCmd": "print(var_dic_list())" - }, - "r": { - "delete_cmd_postfix": ") ", - "delete_cmd_prefix": "rm(", - "library": "var_list.r", - "varRefreshCmd": "cat(var_dic_list()) " - } - }, - "types_to_exclude": [ - "module", - "function", - "builtin_function_or_method", - "instance", - "_Feature" - ], - "window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/importnb/notebooks/path_hooks.ipynb b/src/importnb/notebooks/path_hooks.ipynb new file mode 100644 index 0000000..6baf54e --- /dev/null +++ b/src/importnb/notebooks/path_hooks.ipynb @@ -0,0 +1,252 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# `sys.path_hook` modifiers\n", + "\n", + "Many suggestions for importing notebooks use `sys.meta_paths`, but `importnb` relies on the `sys.path_hooks` to load any notebook in the path. `PathHooksContext` is a base class for the `importnb.Notebook` `SourceFileLoader`." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + " try:\n", + " from .capture import capture_output, CapturedIO\n", + " from .decoder import identity, loads, dedent\n", + " except:\n", + " from capture import capture_output, CapturedIO\n", + " from decoder import identity, loads, dedent\n", + "\n", + " import inspect, sys, ast\n", + " from pathlib import Path\n", + " try: \n", + " from importlib._bootstrap_external import FileFinder\n", + " except:\n", + " #python 3.4\n", + " from importlib.machinery import FileFinder\n", + " \n", + " from contextlib import contextmanager" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + " @contextmanager\n", + " def modify_file_finder_details():\n", + " \"\"\"yield the FileFinder in the sys.path_hooks that loads Python files and assure\n", + " the import cache is cleared afterwards. \n", + " \n", + " Everything goes to shit if the import cache is not cleared.\"\"\"\n", + " \n", + " for id, hook in enumerate(sys.path_hooks):\n", + " try:\n", + " closure = inspect.getclosurevars(hook).nonlocals\n", + " except TypeError: continue\n", + " if issubclass(closure['cls'], FileFinder):\n", + " sys.path_hooks.pop(id)\n", + " details = list(closure['loader_details'])\n", + " yield details\n", + " break\n", + " sys.path_hooks.insert(id, FileFinder.path_hook(*details))\n", + " sys.path_importer_cache.clear()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Update the file_finder details with functions to append and remove the [loader details](https://docs.python.org/3.7/library/importlib.html#importlib.machinery.FileFinder)." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + " def add_path_hooks(loader, extensions, *, position=0):\n", + " \"\"\"Update the FileFinder loader in sys.path_hooks to accomodate a {loader} with the {extensions}\"\"\"\n", + " with modify_file_finder_details() as details:\n", + " details.insert(position, (loader, extensions))\n", + "\n", + " def remove_one_path_hook(loader):\n", + " loader = lazy_loader_cls(loader)\n", + " with modify_file_finder_details() as details:\n", + " _details = list(details)\n", + " for ct, (cls, ext) in enumerate(_details):\n", + " cls = lazy_loader_cls(cls)\n", + " if cls == loader:\n", + " details.pop(ct)\n", + " break" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + " def lazy_loader_cls(loader):\n", + " \"\"\"Extract the loader contents of a lazy loader in the import path.\"\"\"\n", + " try:\n", + " return inspect.getclosurevars(loader).nonlocals.get('cls', loader)\n", + " except:\n", + " return loader" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + " class PathHooksContext:\n", + " def __enter__(self, position=0): \n", + " add_path_hooks(self.prepare(self), self.EXTENSION_SUFFIXES, position=position)\n", + " return self\n", + " \n", + " def __exit__(self, *excepts): remove_one_path_hook(self)\n", + "\n", + " def prepare(self, loader):\n", + " if getattr(self, '_lazy', None): \n", + " try:\n", + " from importlib.util import LazyLoader\n", + " if self._lazy: \n", + " loader = LazyLoader.factory(loader)\n", + " except:\n", + " ImportWarning(\"\"\"LazyLoading is only available in > Python 3.5\"\"\")\n", + " return loader" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + " @contextmanager\n", + " def modify_sys_path(file):\n", + " \"\"\"This is only invoked when using from_resource.\"\"\"\n", + " path = str(Path(file).parent)\n", + " if path not in map(str, map(Path, sys.path)):\n", + " yield sys.path.insert(0, path)\n", + " sys.path = [object for object in sys.path if str(Path(object)) != path]\n", + " else: yield" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Developer" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "TestResults(failed=0, attempted=0)\n" + ] + } + ], + "source": [ + " if __name__ == '__main__':\n", + " try: from utils.export import export\n", + " except: from .utils.export import export\n", + " export('path_hooks.ipynb', '../path_hooks.py')\n", + " import path_hooks\n", + " print(__import__('doctest').testmod(path_hooks))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "p6", + "language": "python", + "name": "other-env" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.3" + }, + "toc": { + "colors": { + "hover_highlight": "#DAA520", + "running_highlight": "#FF0000", + "selected_highlight": "#FFD700" + }, + "moveMenuLeft": true, + "nav_menu": { + "height": "29px", + "width": "252px" + }, + "navigate_menu": true, + "number_sections": true, + "sideBar": true, + "threshold": 4, + "toc_cell": false, + "toc_section_display": "block", + "toc_window_display": false, + "widenNotebook": false + }, + "varInspector": { + "cols": { + "lenName": 16, + "lenType": 16, + "lenVar": 40 + }, + "kernels_config": { + "python": { + "delete_cmd_postfix": "", + "delete_cmd_prefix": "del ", + "library": "var_list.py", + "varRefreshCmd": "print(var_dic_list())" + }, + "r": { + "delete_cmd_postfix": ") ", + "delete_cmd_prefix": "rm(", + "library": "var_list.r", + "varRefreshCmd": "cat(var_dic_list()) " + } + }, + "types_to_exclude": [ + "module", + "function", + "builtin_function_or_method", + "instance", + "_Feature" + ], + "window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/importnb/notebooks/utils/export.ipynb b/src/importnb/notebooks/utils/export.ipynb index 96e355c..c41f893 100644 --- a/src/importnb/notebooks/utils/export.ipynb +++ b/src/importnb/notebooks/utils/export.ipynb @@ -18,9 +18,15 @@ "outputs": [], "source": [ " try:\n", - " from ..decoder import loads, transform_cells\n", + " from ..loader import dedent\n", " except:\n", - " from importnb.decoder import loads, transform_cells" + " from importnb.loader import dedent\n", + " from pathlib import Path\n", + " try:\n", + " from black import format_str\n", + " except:\n", + " format_str = lambda x, i: x\n", + " from json import loads" ] }, { @@ -29,11 +35,17 @@ "metadata": {}, "outputs": [], "source": [ - " from pathlib import Path\n", - " try:\n", - " from black import format_str\n", - " except:\n", - " format_str = lambda x, i: x" + " def block_str(str):\n", + " quotes = '\"\"\"'\n", + " if quotes in str: quotes = \"'''\"\n", + " return \"{quotes}{str}\\n{quotes}\\n\".format(quotes=quotes, str=str)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The export function" ] }, { @@ -43,13 +55,14 @@ "outputs": [], "source": [ " def export(file, to=None): \n", - " \n", + " code = \"\"\"# coding: utf-8\"\"\"\n", " with open(str(file), 'r') as f:\n", - " code = '\\n'.join(\n", - " format_str(''.join(cell['source']), 100)\n", - " for cell in transform_cells(loads(f.read()))['cells']\n", - " if cell['cell_type'] == 'code')\n", - " to and Path(to).with_suffix('.py').write_text(\"# coding: utf-8\\n{}\".format(code))\n", + " for cell in loads(f.read())['cells']:\n", + " if cell['cell_type'] == 'markdown':\n", + " code += '\\n' + block_str(''.join(cell['source']))\n", + " elif cell['cell_type'] == 'code':\n", + " code += '\\n' + dedent(''.join(cell['source']))\n", + " to and Path(to).with_suffix('.py').write_text(format_str(code, 100))\n", " return code " ] }, @@ -68,11 +81,12 @@ " from importnb.utils.export import export\n", "Expecting nothing\n", "ok\n", - "1 items had no tests:\n", + "2 items had no tests:\n", + " export.block_str\n", " export.export\n", "1 items passed all tests:\n", " 1 tests in export\n", - "1 tests in 2 items.\n", + "1 tests in 3 items.\n", "1 passed and 0 failed.\n", "Test passed.\n" ] diff --git a/src/importnb/parameterize.py b/src/importnb/parameterize.py deleted file mode 100644 index c0ac112..0000000 --- a/src/importnb/parameterize.py +++ /dev/null @@ -1,147 +0,0 @@ -# coding: utf-8 -try: - from .loader import Notebook - from .decoder import loads_ast -except: - from loader import Notebook - from decoder import loads_ast -from inspect import getsource - -from types import ModuleType - -"""Are single target `ast.Expr` that will `ast.literal_eval` is a possible parameter. -""" - -from ast import ( - NodeTransformer, - parse, - Assign, - literal_eval, - dump, - fix_missing_locations, - Str, - Tuple, -) - - -class FreeStatementFinder(NodeTransformer): - - def __init__(self, params=None, globals=None): - self.params = params if params is not None else [] - self.globals = globals if globals is not None else {} - - visit_Module = NodeTransformer.generic_visit - - def visit_Assign(FreeStatement, node): - if len(node.targets): - try: - if not getattr(node.targets[0], "id", "_").startswith("_"): - FreeStatement.globals[node.targets[0].id] = literal_eval(node.value) - return - except: - assert True, """The target can not will not literally evaluate.""" - return node - - def generic_visit(self, node): - return node - - def __call__(FreeStatement, nodes): - return FreeStatement.globals, fix_missing_locations(FreeStatement.visit(nodes)) - -"""# `Parameterize` notebooks - -`Parameterize` is callable version of a notebook. It uses `pidgin` to load the `NotebookNode` and evaluates the `FreeStatement`s to discover the signature. -""" - -class Parameterize: - """Parameterize takes a module, filename, or notebook dictionary and returns callable object that parameterizes the notebook module. - - f = Parameterize('parameterize.ipynb') - """ - - def __init__(self, object=None): - from importnb.capture import capture_output - from pathlib import Path - from json import load, loads - - self.object = object - - self.__file__ = None - - if isinstance(object, ModuleType): - self.__file__ = object.__file__ - object = getsource(object) - elif isinstance(object, (Path, str)): - self.__file__ = object - with open(str(object)) as f: - object = f.read() - elif isinstance(object, dict): - ... - else: - raise ValueError("object must be a module, file string, or dict.") - - self.__variables__, self.__ast__ = FreeStatementFinder()(loads_ast(object)) - self.__signature__ = self.vars_to_sig(**self.__variables__) - - def __call__(self, **dict): - self = __import__("copy").copy(self) - self.__dict__.update(self.__variables__) - self.__dict__.update(dict) - exec( - compile(self.__ast__, self.__file__ or "", "exec"), *[self.__dict__] * 2 - ) - return self - - def interact(Parameterize): - """Use the ipywidgets.interact to explore the parameterized notebook.""" - return __import__("ipywidgets").interact(Parameterize) - - @staticmethod - def vars_to_sig(**vars): - """Create a signature for a dictionary of names.""" - from inspect import Parameter, Signature - - return Signature( - [Parameter(str, Parameter.KEYWORD_ONLY, default=vars[str]) for str in vars] - ) - -"""#### Examples that do work -""" - -import sys - -param = "xyz" -extraparam = 42 - -"""#### Examples that do *not* work -""" - -"""Parameters are not created when literal_eval fails.""" -noparam0 = Parameterize - -"""Multiple target assignments are ignored.""" -noparam1, noparam2 = "xyz", 42 - -"""## Developer -""" - -__test__ = dict( - imports=""" - >>> assert callable(f) - """, - default=""" - >>> default = f() - >>> assert default.param == default.noparam1 == 'xyz' and default.noparam2 == 42 - >>> assert all(str not in default.__signature__.parameters for str in ('noparam', 'noparam1', 'noparam2')) - """, - reuse=""" - >>> new = f(param=10) - >>> assert new.param is 10 and new.extraparam is 42""", -) -if __name__ == "__main__": - f = Parameterize(globals().get("__file__", "parameterize.ipynb")) - __import__("doctest").testmod(verbose=1) - -if __name__ == "__main__": - # export('parameterize.ipynb', '../parameterize.py') - __import__("doctest").testmod() diff --git a/src/importnb/path_hooks.py b/src/importnb/path_hooks.py new file mode 100644 index 0000000..9480488 --- /dev/null +++ b/src/importnb/path_hooks.py @@ -0,0 +1,119 @@ +# coding: utf-8 +"""# `sys.path_hook` modifiers + +Many suggestions for importing notebooks use `sys.meta_paths`, but `importnb` relies on the `sys.path_hooks` to load any notebook in the path. `PathHooksContext` is a base class for the `importnb.Notebook` `SourceFileLoader`. +""" + +try: + from .capture import capture_output, CapturedIO + from .decoder import identity, loads, dedent +except: + from capture import capture_output, CapturedIO + from decoder import identity, loads, dedent + +import inspect, sys, ast +from pathlib import Path + +try: + from importlib._bootstrap_external import FileFinder +except: + # python 3.4 + from importlib.machinery import FileFinder + +from contextlib import contextmanager + + +@contextmanager +def modify_file_finder_details(): + """yield the FileFinder in the sys.path_hooks that loads Python files and assure + the import cache is cleared afterwards. + + Everything goes to shit if the import cache is not cleared.""" + + for id, hook in enumerate(sys.path_hooks): + try: + closure = inspect.getclosurevars(hook).nonlocals + except TypeError: + continue + if issubclass(closure["cls"], FileFinder): + sys.path_hooks.pop(id) + details = list(closure["loader_details"]) + yield details + break + sys.path_hooks.insert(id, FileFinder.path_hook(*details)) + sys.path_importer_cache.clear() + + +"""Update the file_finder details with functions to append and remove the [loader details](https://docs.python.org/3.7/library/importlib.html#importlib.machinery.FileFinder). +""" + + +def add_path_hooks(loader, extensions, *, position=0): + """Update the FileFinder loader in sys.path_hooks to accomodate a {loader} with the {extensions}""" + with modify_file_finder_details() as details: + details.insert(position, (loader, extensions)) + + +def remove_one_path_hook(loader): + loader = lazy_loader_cls(loader) + with modify_file_finder_details() as details: + _details = list(details) + for ct, (cls, ext) in enumerate(_details): + cls = lazy_loader_cls(cls) + if cls == loader: + details.pop(ct) + break + + +def lazy_loader_cls(loader): + """Extract the loader contents of a lazy loader in the import path.""" + try: + return inspect.getclosurevars(loader).nonlocals.get("cls", loader) + except: + return loader + + +class PathHooksContext: + + def __enter__(self, position=0): + add_path_hooks(self.prepare(self), self.EXTENSION_SUFFIXES, position=position) + return self + + def __exit__(self, *excepts): + remove_one_path_hook(self) + + def prepare(self, loader): + if getattr(self, "_lazy", None): + try: + from importlib.util import LazyLoader + + if self._lazy: + loader = LazyLoader.factory(loader) + except: + ImportWarning("""LazyLoading is only available in > Python 3.5""") + return loader + + +@contextmanager +def modify_sys_path(file): + """This is only invoked when using from_resource.""" + path = str(Path(file).parent) + if path not in map(str, map(Path, sys.path)): + yield sys.path.insert(0, path) + sys.path = [object for object in sys.path if str(Path(object)) != path] + else: + yield + + +"""# Developer +""" + +if __name__ == "__main__": + try: + from utils.export import export + except: + from .utils.export import export + export("path_hooks.ipynb", "../path_hooks.py") + import path_hooks + + print(__import__("doctest").testmod(path_hooks)) diff --git a/src/importnb/tests/test_importnb.ipynb b/src/importnb/tests/test_importnb.ipynb index e36de71..a0d748d 100644 --- a/src/importnb/tests/test_importnb.ipynb +++ b/src/importnb/tests/test_importnb.ipynb @@ -14,10 +14,13 @@ } ], "source": [ - "from importnb import Notebook, reload, Lazy, Partial, load_ipython_extension, unload_ipython_extension, Execute\n", + "from importnb import Notebook, reload, load_ipython_extension, unload_ipython_extension, Execute\n", "from pathlib import Path\n", "import shutil, os, functools, sys\n", "from pytest import fixture, mark\n", + "from functools import partial\n", + "Lazy = partial(Notebook, lazy=True)\n", + "Partial = partial(Notebook, exceptions=BaseException)\n", "import warnings\n", "try: __IPYTHON__\n", "except: __IPYTHON__ = False \n", @@ -62,6 +65,7 @@ " new_code_cell(\"\"\"foo = 42\\nassert {}\\nbar= 100\"\"\".format(str)),\n", " new_code_cell(\"\"\"print(foo)\"\"\"),\n", " new_markdown_cell(\"\"\"Markdown paragraph\"\"\"),\n", + " new_code_cell(\"\"\"_repr_markdown_ = lambda: 'a custom repr {foo}'.format(foo=foo)\"\"\"),\n", " ]))" ] }, @@ -167,7 +171,7 @@ " foobar = Notebook(stdout=True).from_filename('foobar.ipynb')\n", " assert foobar.foo == 42 and foobar.bar == 100\n", " assert foobar.__name__ != '__main__'\n", - " assert foobar.__output__.stdout\n", + " assert foobar._capture.stdout\n", " assert foobar.__doc__.strip().startswith(\"\"\"This is the docstring.\"\"\")" ] }, @@ -182,7 +186,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -191,16 +195,16 @@ " foobar = Execute(stdout=True).from_filename('foobar.ipynb')\n", " assert foobar.foo == 42 and foobar.bar == 100\n", " assert foobar.__name__ != '__main__'\n", - " assert foobar.__notebook__\n", + " assert foobar._notebook\n", " assert any(\n", - " any(set(output.items()).issubset(new_stream(\"42\\n\").items()) for output in object['outputs']) for object in foobar.__notebook__['cells']\n", + " any(set(output.items()).issubset(new_stream(\"42\\n\").items()) for output in object.get('outputs', [])) for object in foobar._notebook['cells']\n", " )\n", " assert foobar.__doc__.strip().startswith(\"\"\"This is the docstring.\"\"\")" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -215,7 +219,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -223,7 +227,7 @@ " foobar = Notebook('__main__', stdout=True).from_filename('foobar.ipynb')\n", " assert foobar.foo == 42 and foobar.bar == 100\n", " assert foobar.__name__ == '__main__'\n", - " assert foobar.__output__.stdout\n", + " assert foobar._capture.stdout\n", " " ] }, @@ -239,7 +243,7 @@ " \n", " from importnb.execute import Parameterize\n", " \n", - " f = Parameterize().from_filename(foobar.__file__)\n", + " f = Parameterize(stdout=True).from_filename(foobar.__file__)\n", " \n", " foobar = f()\n", " \n", @@ -247,7 +251,12 @@ " \n", " foobar = f(foo=\"something\", bar=0)\n", " print(foobar.foo, foobar.bar)\n", - " assert foobar.foo == \"something\" and foobar.bar == 0" + " assert foobar.foo == \"something\" and foobar.bar == 0\n", + " \n", + " assert any(\n", + " any(set(output.items()).issubset(new_stream(\"something\\n\").items()) for output in object.get('outputs', [])) for object in foobar._notebook['cells']\n", + " )\n", + " # assert foobar.__doc__.strip().startswith(\"\"\"This is the docstring.\"\"\")" ] }, { @@ -274,7 +283,7 @@ " # I don't think i can test stderr with pytest\n", " with Notebook(stdout=True, stderr=True):\n", " import foobar\n", - " out = foobar.__output__\n", + " out = foobar._capture\n", " assert foobar.foo == 42 and foobar.bar == 100\n", " assert out.stdout\n", "# assert out.stderr\n", @@ -494,7 +503,7 @@ " with Partial():\n", " from a_test_package import failure\n", " \n", - " assert isinstance(failure.__exception__, AssertionError), \"\"\"\n", + " assert isinstance(failure._exception, AssertionError), \"\"\"\n", " The wrong error was returned likely because of importnb.\"\"\"\n", "\n", " from traceback import print_tb\n", @@ -502,9 +511,30 @@ " s = StringIO()\n", " with open(failure.__file__, 'r') as f:\n", " line = list(i for i, line in enumerate(f.read().splitlines()) if 'assert False' in line)[0] + 1\n", - " print_tb(failure.__exception__.__traceback__, file=s)\n", + " print_tb(failure._exception.__traceback__, file=s)\n", " assert \"\"\"a_test_package/failure.ipynb\", line {}, in \\n\"\"\".format(line) in s.getvalue(), \"\"\"Traceback is not satisfied\"\"\"" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/src/importnb/tests/test_module.ipynb b/src/importnb/tests/test_module.ipynb deleted file mode 100644 index ab68263..0000000 --- a/src/importnb/tests/test_module.ipynb +++ /dev/null @@ -1,64 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - " from importnb import *" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - " try:\n", - " from pytest import fixture\n", - " from importlib import import_module\n", - " @fixture\n", - " def module(): \n", - " return import_module(__name__)\n", - " except: ..." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - " def test_all():\n", - "\n", - " assert all(\n", - " object in globals()\n", - " for object in (\n", - " 'Notebook', 'Partial', 'reload', 'Parameterize', 'Lazy'\n", - " ))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "p6", - "language": "python", - "name": "other-env" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/importnb/tests/test_unittests.ipynb b/src/importnb/tests/test_unittests.ipynb index fbc0c95..b630599 100644 --- a/src/importnb/tests/test_unittests.ipynb +++ b/src/importnb/tests/test_unittests.ipynb @@ -39,6 +39,10 @@ " __import__('subprocess').check_call(\"pip install importnb\".split())\n", " from importnb import *\n", "\n", + " from functools import partial\n", + " Lazy = partial(Notebook, lazy=True)\n", + " Partial = partial(Notebook, exceptions=BaseException)\n", + "\n", " del testmod\n", " __file__ = globals().get('__file__', 'unittests.ipynb')" ] @@ -124,14 +128,14 @@ " \n", " def test_exception(Test):\n", " assert Test.failure.a is 42\n", - " assert isinstance(Test.failure.__exception__, BaseException)\n", + " assert isinstance(Test.failure._exception, BaseException)\n", " assert not hasattr(Test.failure, 'b')\n", " \n", " def test_traceback(Test):\n", " from traceback import print_tb\n", " from io import StringIO\n", " s = StringIO()\n", - " print_tb(Test.failure.__exception__.__traceback__, file=s)\n", + " print_tb(Test.failure._exception.__traceback__, file=s)\n", " assert \"\"\"tests/failure.ipynb\", line 22, in \\n \" assert False\\\\n\"\"\" in s.getvalue(), \"\"\"Traceback is not satisfied\"\"\"" ] }, diff --git a/src/importnb/utils/export.py b/src/importnb/utils/export.py index 1753f8e..e3d3f46 100644 --- a/src/importnb/utils/export.py +++ b/src/importnb/utils/export.py @@ -7,28 +7,41 @@ """ try: - from ..decoder import loads, transform_cells + from ..loader import dedent except: - from importnb.decoder import loads, transform_cells - + from importnb.loader import dedent from pathlib import Path try: from black import format_str except: format_str = lambda x, i: x +from json import loads + + +def block_str(str): + quotes = '"""' + if quotes in str: + quotes = "'''" + return "{quotes}{str}\n{quotes}\n".format(quotes=quotes, str=str) + + +"""The export function +""" -def export(file, to=None): +def export(file, to=None): + code = """# coding: utf-8""" with open(str(file), "r") as f: - code = "\n".join( - format_str("".join(cell["source"]), 100) - for cell in transform_cells(loads(f.read()))["cells"] - if cell["cell_type"] == "code" - ) - to and Path(to).with_suffix(".py").write_text("# coding: utf-8\n{}".format(code)) + for cell in loads(f.read())["cells"]: + if cell["cell_type"] == "markdown": + code += "\n" + block_str("".join(cell["source"])) + elif cell["cell_type"] == "code": + code += "\n" + dedent("".join(cell["source"])) + to and Path(to).with_suffix(".py").write_text(format_str(code, 100)) return code + if __name__ == "__main__": export("export.ipynb", "../../utils/export.py") try: @@ -36,4 +49,3 @@ def export(file, to=None): except: from . import export as this __import__("doctest").testmod(this, verbose=2) - diff --git a/src/importnb/utils/ipython.py b/src/importnb/utils/ipython.py index 00f7df6..f162816 100644 --- a/src/importnb/utils/ipython.py +++ b/src/importnb/utils/ipython.py @@ -4,10 +4,12 @@ from pathlib import Path import json + def get_config(): ip = get_ipython() return Path(ip.profile_dir.location if ip else paths.locate_profile()) / "ipython_config.json" + def load_config(): location = get_config() try: @@ -24,6 +26,7 @@ def load_config(): return config, location + def install(ip=None): config, location = load_config() @@ -33,10 +36,12 @@ def install(ip=None): with location.open("w") as file: json.dump(config, file) + def installed(): config = load_config() return "importnb.utils.ipython" in config.get("InteractiveShellApp", {}).get("extensions", []) + def uninstall(ip=None): config, location = load_config() @@ -49,15 +54,16 @@ def uninstall(ip=None): with location.open("w") as file: json.dump(config, file) + def load_ipython_extension(ip): from ..loader import Notebook Notebook().__enter__(position=-1) + if __name__ == "__main__": try: from ..loader import export except: from importnb.loader import export export("ipython.ipynb", "../../utils/ipython.py") - diff --git a/src/importnb/utils/nbdoctest.py b/src/importnb/utils/nbdoctest.py index 0ebb762..bc1623a 100644 --- a/src/importnb/utils/nbdoctest.py +++ b/src/importnb/utils/nbdoctest.py @@ -8,6 +8,7 @@ except: from importnb import Notebook + def _test(): parser = argparse.ArgumentParser(description="doctest runner") parser.add_argument( @@ -66,6 +67,7 @@ def _test(): return 1 return 0 + if __name__ == "__main__": _test() diff --git a/src/importnb/utils/pytest_plugin.py b/src/importnb/utils/pytest_plugin.py index 05b795e..2ffd354 100644 --- a/src/importnb/utils/pytest_plugin.py +++ b/src/importnb/utils/pytest_plugin.py @@ -27,10 +27,10 @@ def collect(self): with loader(): return super().collect() + if __name__ == "__main__": try: from ..loader import export except: from importnb.loader import export export("pytest_plugin.ipynb", "../../utils/pytest_plugin.py") - diff --git a/src/importnb/utils/setup.py b/src/importnb/utils/setup.py index 80d92e1..3fafd14 100644 --- a/src/importnb/utils/setup.py +++ b/src/importnb/utils/setup.py @@ -81,10 +81,10 @@ def find_modules(self): return modules + if __name__ == "__main__": try: from ..loader import export except: from importnb.loader import export export("setup.ipynb", "../../utils/setup.py") - diff --git a/src/importnb/utils/watch.py b/src/importnb/utils/watch.py index fff88c4..2990663 100644 --- a/src/importnb/utils/watch.py +++ b/src/importnb/utils/watch.py @@ -13,6 +13,7 @@ import os from watchdog.tricks import ShellCommandTrick + class ModuleTrick(ShellCommandTrick): """ModuleTrick is a watchdog trick that """ @@ -32,10 +33,10 @@ def on_any_event(self, event): except AttributeError: ... + if __name__ == "__main__": try: from ..loader import export except: from importnb.loader import export export("watch.ipynb", "../../utils/watch.py") -