From 6f28cc8bd4fed9e97ac5edb41f302c1f9b28aae1 Mon Sep 17 00:00:00 2001 From: Angus Hollands Date: Wed, 11 May 2022 10:46:17 +0100 Subject: [PATCH] Feat: install import hook by default --- literary-hook.pth | 2 + pdm.lock | 26 ++++---- pyproject.toml | 7 ++- src/literary/config/__init__.ipynb | 52 ++++++++-------- src/literary/hook/__init__.ipynb | 40 +++++++----- src/literary/hook/importer.ipynb | 93 ++++++++++++++++++++++++++-- src/literary/hook/loader.ipynb | 20 +----- src/literary/module/__init__.ipynb | 47 +++++++------- src/literary/notebook/__init__.ipynb | 23 ------- 9 files changed, 182 insertions(+), 128 deletions(-) create mode 100644 literary-hook.pth diff --git a/literary-hook.pth b/literary-hook.pth new file mode 100644 index 0000000..fab6164 --- /dev/null +++ b/literary-hook.pth @@ -0,0 +1,2 @@ +# Install notebook import hook +import literary.hook; literary.hook.install_import_hook() \ No newline at end of file diff --git a/pdm.lock b/pdm.lock index 5a72d8f..0f1ad96 100644 --- a/pdm.lock +++ b/pdm.lock @@ -339,18 +339,18 @@ dependencies = [ [[package]] name = "literary" -version = "3.0.2" +version = "4.0.0a0" requires_python = ">=3.7" summary = "Literate package development with Jupyter" dependencies = [ - "astunparse~=1.6; python_version < \"3.9\"", - "ipython", - "jupyter-core<4.8,>=4.7", - "nbclient<0.6,>=0.5", - "nbconvert<7,>=6", - "nbformat[fast]<6,>=4", - "traitlets<6,>=5", - "typing-extensions", + "astunparse>=1.6; python_version < \"3.9\"", + "ipython>=7.33.0", + "jupyter-core>=4.7", + "nbclient>=0.5.12", + "nbconvert>=6.0", + "nbformat[fast]>=5", + "traitlets>=5", + "typing-extensions>=3.10", ] [[package]] @@ -751,7 +751,7 @@ summary = "Backport of pathlib-compatible object wrapper for zip files" [metadata] lock_version = "3.1" -content_hash = "sha256:c89ef8e0908db6d1870e8bb59298a6df0f46af23f11fc16b6d4605237dcb4269" +content_hash = "sha256:b258efddcbd536fc8afce5fef1fc663198349cf48069caf43e224964b71e5a91" [metadata.files] "anyio 3.5.0" = [ @@ -984,9 +984,9 @@ content_hash = "sha256:c89ef8e0908db6d1870e8bb59298a6df0f46af23f11fc16b6d4605237 {file = "jupyterlab_server-2.13.0-py3-none-any.whl", hash = "sha256:fc9e86d4e7c4b139de59b0a96b53071e670bee1ed106a3389daecd68f1221aeb"}, {file = "jupyterlab_server-2.13.0.tar.gz", hash = "sha256:2040298a133458aa22f287a877d6bb91ff973f6298d562264f9f7b75e92a5ace"}, ] -"literary 3.0.2" = [ - {file = "literary-3.0.2-py3-none-any.whl", hash = "sha256:e27ae87d2c5b8a61466eb967c054550c2e538165a91cc7bc5f2e805841fb8c9a"}, - {file = "literary-3.0.2.tar.gz", hash = "sha256:a199a72006e674c301bc9303db9779515d359bfce07b5629ffbed8e8cc373bb7"}, +"literary 4.0.0a0" = [ + {file = "literary-4.0.0a0-py3-none-any.whl", hash = "sha256:aeb1156c67824dd356bffffec976fc81f2de88f00bd12a73d4a38881a3e76b63"}, + {file = "literary-4.0.0a0.tar.gz", hash = "sha256:1431bcaca7977c2423ef0a025ffdcb426679afdc004cb0ec6baf741888f558d0"}, ] "markupsafe 2.1.1" = [ {file = "MarkupSafe-2.1.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812"}, diff --git a/pyproject.toml b/pyproject.toml index f83116c..7a88c8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "literary" -version = "4.0.0a0" +version = "4.0.0a1" description = "Literate package development with Jupyter" authors = [ {name = "Angus Hollands", email = "goosey15@gmail.com"}, @@ -64,14 +64,15 @@ exclude = ["lib"] # Include generated package for wheels [tool.hatch.build.targets.wheel.force-include] +# Unpack built lib into root "lib" = "/" - +# Install hook into environment +"literary-hook.pth" = "literary-hook.pth" [tool.hatch.build.targets.wheel.shared-data] "share/jupyter/nbconvert/templates/literary/conf.json" = "share/jupyter/nbconvert/templates/literary/conf.json" "share/jupyter/nbconvert/templates/literary/index.py.j2" = "share/jupyter/nbconvert/templates/literary/index.py.j2" - [tool.hatch.build.targets.wheel.hooks.literary] dependencies = ["literary-build-hatch>=0.2.0a3"] diff --git a/src/literary/config/__init__.ipynb b/src/literary/config/__init__.ipynb index ac9f1c6..0e86cb5 100644 --- a/src/literary/config/__init__.ipynb +++ b/src/literary/config/__init__.ipynb @@ -10,19 +10,28 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 6, "id": "acoustic-prompt", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The literary.module extension is already loaded. To reload it, use:\n", + " %reload_ext literary.module\n" + ] + } + ], "source": [ "%load_ext literary.module" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 7, "id": "front-notion", "metadata": { "tags": [ @@ -53,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 8, "id": "8ed360c7-eef6-4707-804a-744225ecdd79", "metadata": { "tags": [ @@ -70,12 +79,12 @@ "id": "d92cece7-9d15-4986-9293-c4b806a99771", "metadata": {}, "source": [ - "To find the config file, we simply perform a depth-first search of each given path, and return the first file with the appropriate stem." + "To find the config file, we simply perform a breadth-first search of each given path, and return the first file with the appropriate stem." ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 13, "id": "environmental-cooperation", "metadata": { "tags": [ @@ -85,24 +94,18 @@ "outputs": [], "source": [ "@lru_cache()\n", - "def find_literary_config(path, *additional_paths) -> Path:\n", + "def find_literary_config(path) -> Path:\n", " \"\"\"Load the configuration for the current Literary project.\n", "\n", - " :param search_paths: starting search paths\n", " :return:\n", " \"\"\"\n", - " visited = set()\n", - "\n", - " for top_level_path in [path, *additional_paths]:\n", - " for search_path in (top_level_path, *top_level_path.parents):\n", - " # Avoid re-visiting paths\n", - " if search_path in visited:\n", - " break\n", - " visited.add(search_path)\n", - "\n", - " # Look for any config file\n", - " for p in search_path.glob(f\"{CONFIG_FILE_STEM}.*\"):\n", - " return p\n", + " # Look for any config file\n", + " for p in path.glob(f\"{CONFIG_FILE_STEM}.*\"):\n", + " return p\n", + " \n", + " # Visit parent\n", + " if path.parents:\n", + " return find_literary_config(path.parent)\n", "\n", " raise FileNotFoundError(\"Couldn't find config file\")" ] @@ -117,7 +120,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 14, "id": "131149c5-b284-4765-b079-abbd2054b350", "metadata": { "tags": [ @@ -135,14 +138,11 @@ " for loader_cls in JSONFileConfigLoader, PyFileConfigLoader:\n", " loader = loader_cls(path.name, str(path.parent))\n", " try:\n", - " config = loader.load_config()\n", - " break\n", + " return loader.load_config()\n", " except ConfigFileNotFound:\n", " continue\n", - " else:\n", - " raise ValueError(f\"{path!r} was not a recognised config file\")\n", "\n", - " return config" + " raise ValueError(f\"{path!r} was not a recognised config file\")" ] } ], diff --git a/src/literary/hook/__init__.ipynb b/src/literary/hook/__init__.ipynb index 31fbaa4..bd51799 100644 --- a/src/literary/hook/__init__.ipynb +++ b/src/literary/hook/__init__.ipynb @@ -22,18 +22,24 @@ "outputs": [], "source": [ "from .finder import extend_file_finder\n", - "from ..config import load_literary_config, find_literary_config\n", "\n", - "import functools\n", "import sys\n", "import traceback\n", "import pathlib" ] }, + { + "cell_type": "markdown", + "id": "fdab96e1-95b7-47d7-bba1-de073ea1b1a2", + "metadata": {}, + "source": [ + "We only want to lazy load once, to ensure that we always get the bootstrapping Literary's import hook" + ] + }, { "cell_type": "code", "execution_count": 3, - "id": "14cdff56-cb55-47d7-99f5-15ea91c84a09", + "id": "392c9424-1934-4a38-a636-f1f8c9623149", "metadata": { "tags": [ "export" @@ -41,7 +47,21 @@ }, "outputs": [], "source": [ - "load_cached_config = functools.lru_cache()(load_literary_config)" + "def notebook_loader_factory(fullname, path):\n", + " try:\n", + " factory = notebook_loader_factory.factory\n", + " except AttributeError:\n", + " # Re-entrant guard\n", + " def noop_loader(fullname, path):\n", + " return None\n", + " notebook_loader_factory.factory = noop_loader\n", + " \n", + " # Actual importer\n", + " from .importer import get_loader\n", + " notebook_loader_factory.factory = get_loader\n", + " factory = get_loader\n", + " \n", + " return factory(fullname, path)" ] }, { @@ -56,17 +76,7 @@ "outputs": [], "source": [ "def install_import_hook(set_except_hook=True):\n", - " # Inject notebook loader into path_hooks\n", - " def create_notebook_loader(fullname, path):\n", - " from .importer import NotebookImporter\n", - "\n", - " config = load_cached_config(\n", - " find_literary_config(pathlib.Path(path))\n", - " )\n", - " importer = NotebookImporter(config=config)\n", - " return importer.get_loader(fullname, path)\n", - "\n", - " extend_file_finder((create_notebook_loader, [\".ipynb\"]),)\n", + " extend_file_finder((notebook_loader_factory, [\".ipynb\"]),)\n", "\n", " if set_except_hook:\n", " sys.excepthook = traceback.print_exception" diff --git a/src/literary/hook/importer.ipynb b/src/literary/hook/importer.ipynb index 5d4e246..8cb29de 100644 --- a/src/literary/hook/importer.ipynb +++ b/src/literary/hook/importer.ipynb @@ -14,19 +14,28 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 4, "id": "convenient-james", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The literary.module extension is already loaded. To reload it, use:\n", + " %reload_ext literary.module\n" + ] + } + ], "source": [ "%load_ext literary.module" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "id": "atomic-restoration", "metadata": { "tags": [ @@ -35,10 +44,13 @@ }, "outputs": [], "source": [ + "from functools import lru_cache\n", + "from pathlib import Path\n", "from nbconvert import Exporter\n", "from traitlets import Instance, Type, default\n", "from traitlets.config import Configurable\n", "\n", + "from ..config import find_literary_config, load_literary_config\n", "from ..transpile.exporter import LiteraryExporter\n", "from .loader import NotebookLoader" ] @@ -53,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "4bd4893f-d373-4161-80b3-1ce334492a15", "metadata": { "tags": [ @@ -67,9 +79,17 @@ " exporter_class = Type(LiteraryExporter).tag(config=True)" ] }, + { + "cell_type": "markdown", + "id": "c79c6508-f45e-4815-affe-8291b809cd16", + "metadata": {}, + "source": [ + "By default, we'll create an instance of `exporter_class`" + ] + }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "c2b8b55d-8d52-49d4-8c99-97731232187c", "metadata": { "tags": [ @@ -84,9 +104,17 @@ " return self.exporter_class(parent=self)" ] }, + { + "cell_type": "markdown", + "id": "6c6fcdac-e367-443e-b0f1-367bc1820ddb", + "metadata": {}, + "source": [ + "We will implement a method to build a `NotebookLoader` object using the current exporter" + ] + }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "id": "f8172c3a-00b6-41eb-80f9-bfbe5a67a5ad", "metadata": { "tags": [ @@ -101,6 +129,59 @@ " exporter = self.exporter_class(parent=self)\n", " return NotebookLoader(fullname, path, exporter=self.exporter)" ] + }, + { + "cell_type": "markdown", + "id": "3237d24c-960d-4e31-bc0f-8f911d7a18f1", + "metadata": {}, + "source": [ + "## Loader factory\n", + "To avoid a startup penalty, we want this module to be imported lazily. We should also load the configuation lazily" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e3659fa-dd6c-4f42-897d-49cf75badfc7", + "metadata": { + "tags": [ + "export" + ] + }, + "outputs": [], + "source": [ + "@lru_cache()\n", + "def load_cached_config(path):\n", + " return load_literary_config(path)" + ] + }, + { + "cell_type": "markdown", + "id": "498179f1-f812-4a6c-862c-f8c3c6dcb16f", + "metadata": {}, + "source": [ + "However, we also do not want to have to lookup relative imports once the import hook has been installed. It this were to happen, the import hook itself may start resolving against the local notebooks (and things would break)! So, we provide a factory function here that can be imported once on demand by the hook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1fc2bb1-4d07-40a8-8f31-02e425661eda", + "metadata": { + "tags": [ + "export" + ] + }, + "outputs": [], + "source": [ + "def get_loader(fullname, path):\n", + " try:\n", + " config_path = find_literary_config(Path(path))\n", + " except FileNotFoundError:\n", + " return None\n", + " importer = NotebookImporter(config=load_cached_config(config_path))\n", + " return importer.get_loader(fullname, path)" + ] } ], "metadata": { diff --git a/src/literary/hook/loader.ipynb b/src/literary/hook/loader.ipynb index 8447fe1..be250b8 100644 --- a/src/literary/hook/loader.ipynb +++ b/src/literary/hook/loader.ipynb @@ -17,12 +17,6 @@ "execution_count": 1, "id": "a2115a46-2725-45c4-9a9f-75eee74e0869", "metadata": { - "execution": { - "iopub.execute_input": "2021-05-18T21:00:43.643548Z", - "iopub.status.busy": "2021-05-18T21:00:43.640394Z", - "iopub.status.idle": "2021-05-18T21:00:43.781188Z", - "shell.execute_reply": "2021-05-18T21:00:43.780138Z" - }, "tags": [] }, "outputs": [], @@ -38,12 +32,6 @@ "execution_count": 2, "id": "c7ff6b8f-9153-41d9-97c3-90f9c5e055ff", "metadata": { - "execution": { - "iopub.execute_input": "2021-05-18T21:00:43.786993Z", - "iopub.status.busy": "2021-05-18T21:00:43.785810Z", - "iopub.status.idle": "2021-05-18T21:00:43.789256Z", - "shell.execute_reply": "2021-05-18T21:00:43.788416Z" - }, "tags": [ "export" ] @@ -51,6 +39,7 @@ "outputs": [], "source": [ "import linecache\n", + "import nbformat\n", "from importlib.machinery import SourcelessFileLoader" ] }, @@ -59,12 +48,6 @@ "execution_count": 3, "id": "5bcde51b-dda5-4613-afbc-de78ba2232e8", "metadata": { - "execution": { - "iopub.execute_input": "2021-05-18T21:00:43.800505Z", - "iopub.status.busy": "2021-05-18T21:00:43.799286Z", - "iopub.status.idle": "2021-05-18T21:00:43.802000Z", - "shell.execute_reply": "2021-05-18T21:00:43.802986Z" - }, "tags": [ "export" ] @@ -95,7 +78,6 @@ " return compile(body, path, \"exec\")\n", "\n", " def get_transpiled_source(self, path: str):\n", - " import nbformat\n", " nb = nbformat.read(path, as_version=nbformat.NO_CONVERT)\n", " body, resources = self._exporter.from_notebook_node(nb)\n", " return body" diff --git a/src/literary/module/__init__.ipynb b/src/literary/module/__init__.ipynb index 63e1db8..3ae44ff 100644 --- a/src/literary/module/__init__.ipynb +++ b/src/literary/module/__init__.ipynb @@ -14,48 +14,45 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 5, "id": "grave-racing", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The literary.module extension is already loaded. To reload it, use:\n", + " %reload_ext literary.module\n" + ] + } + ], "source": [ "%load_ext literary.module" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 6, "id": "grave-finding", "metadata": { "tags": [ "export" ] }, - "outputs": [ - { - "ename": "ImportError", - "evalue": "attempted relative import with no known parent package", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/tmp/ipykernel_3162281/1607555708.py\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mpathlib\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPath\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhook\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0minstall_import_hook\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpatch\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpatch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mImportError\u001b[0m: attempted relative import with no known parent package" - ] - } - ], + "outputs": [], "source": [ "import sys\n", + "import warnings\n", "from pathlib import Path\n", - "from ..hook import install_import_hook\n", "from ..transpile.patch import patch" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "48f56bbb-2398-4bd1-b16b-af6e7ca15c27", "metadata": { "tags": [ @@ -91,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "classical-shoulder", "metadata": { "tags": [ @@ -106,15 +103,19 @@ " added to `sys.path`.\n", "\n", " :param ipython: IPython shell instance\n", - " \"\"\"\n", - " # Ensure that notebook hook is installed\n", - " install_import_hook()\n", - " \n", + " \"\"\" \n", " cwd = Path.cwd()\n", " \n", " # Ensure CWD is not on sys.path\n", " sys.path = [p for p in sys.path if Path(p).resolve() != cwd]\n", "\n", + " # Identify which package this belongs to\n", + " package = determine_package_name(cwd) \n", + " if package is None:\n", + " warnings.warn(f\"Couldn't determine the package name for the current working directory {cwd}. \"\n", + " f\"This might be because the current project has not been installed in editable mode.\")\n", + " \n", + " \n", " # Set `__package__` for consumer notebook\n", " ipython.user_ns.update(\n", " {\n", diff --git a/src/literary/notebook/__init__.ipynb b/src/literary/notebook/__init__.ipynb index bcb95bb..64b0121 100644 --- a/src/literary/notebook/__init__.ipynb +++ b/src/literary/notebook/__init__.ipynb @@ -17,12 +17,6 @@ "execution_count": 1, "id": "grave-racing", "metadata": { - "execution": { - "iopub.execute_input": "2021-05-18T21:00:42.279792Z", - "iopub.status.busy": "2021-05-18T21:00:42.278794Z", - "iopub.status.idle": "2021-05-18T21:00:42.421234Z", - "shell.execute_reply": "2021-05-18T21:00:42.422196Z" - }, "tags": [] }, "outputs": [], @@ -35,19 +29,12 @@ "execution_count": 2, "id": "grave-finding", "metadata": { - "execution": { - "iopub.execute_input": "2021-05-18T21:00:42.429509Z", - "iopub.status.busy": "2021-05-18T21:00:42.428281Z", - "iopub.status.idle": "2021-05-18T21:00:42.431224Z", - "shell.execute_reply": "2021-05-18T21:00:42.432229Z" - }, "tags": [ "export" ] }, "outputs": [], "source": [ - "from ..hook import install_import_hook\n", "from ..transpile.patch import patch" ] }, @@ -56,12 +43,6 @@ "execution_count": 3, "id": "classical-shoulder", "metadata": { - "execution": { - "iopub.execute_input": "2021-05-18T21:00:42.437797Z", - "iopub.status.busy": "2021-05-18T21:00:42.436839Z", - "iopub.status.idle": "2021-05-18T21:00:42.439775Z", - "shell.execute_reply": "2021-05-18T21:00:42.439132Z" - }, "tags": [ "export" ] @@ -75,10 +56,6 @@ "\n", " :param ipython: IPython shell instance\n", " \"\"\"\n", - " # Ensure that notebook hook is installed\n", - " install_import_hook()\n", - "\n", - " # Set `__package__` for consumer notebook\n", " ipython.user_ns.update(\n", " {\n", " \"patch\": patch,\n",