diff --git a/.github/workflows/build_site.yml b/.github/workflows/build_site.yml index b3a89731..0dc5c60d 100644 --- a/.github/workflows/build_site.yml +++ b/.github/workflows/build_site.yml @@ -10,18 +10,18 @@ jobs: Build-website: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # Adds keys to work with other repositories used in this lesson (e.g., UCL/github-example) - - uses: webfactory/ssh-agent@v0.5.4 + - uses: webfactory/ssh-agent@v0.9.0 with: ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - - uses: actions/cache@v2 # FIXME: add apt(latex) + - uses: actions/cache@v4 # FIXME: add apt(latex) with: path: vendor/bundle key: ${{ runner.os }}-gems-${{ hashFiles('**/Gemfile') }} restore-keys: | ${{ runner.os }}-gems- - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt', '.github/python/requirements.txt') }} @@ -31,23 +31,30 @@ jobs: uses: DanySK/setup-texlive-action@0.1.1 - id: setup-python name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.12 - name: Install dependencies - uses: py-actions/py-dependency-install@v2 - with: - path: ".github/python/requirements.txt" + run: python -m pip install -r .github/python/requirements.txt - name: Building notes run: make ready - - name: Builds website - uses: helaili/jekyll-action@v2 + - uses: ruby/setup-ruby@v1 with: - target_branch: 'gh-pages' - build_only: true - - name: Build website and deploy - if: github.ref == 'refs/heads/main' - uses: helaili/jekyll-action@v2 + ruby-version: '3.3' + - name: Install bundler and dependencies + run: | + gem install bundler + bundle install + - name: Build website + run: bundle exec jekyll build + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 with: - target_branch: 'gh-pages' - token: ${{ secrets.GITHUB_TOKEN }} + path: _site + - name: Deploy to GitHub Pages + if: > + success() + && github.ref == 'refs/heads/main' + && github.repository == 'UCL/rsd-engineeringcourse' + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 7d18429b..a33f613f 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,6 @@ master.zip session04/greetings/Greetings.egg-info/ session04/greetings/doc/ session04/greetings/scripts/ +Gemfile.lock +.env/ + diff --git a/Gemfile b/Gemfile index 6ad3a913..0688490b 100644 --- a/Gemfile +++ b/Gemfile @@ -1,5 +1,6 @@ source 'https://rubygems.org' -gem 'jekyll', '~> 4.2.1' +gem "jekyll" gem 'kramdown', '~> 2.3.1' gem "jekyll-remote-theme" +gem "jekyll-sass-converter" diff --git a/README.md b/README.md index fd6194e4..6ae34ec7 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ $ docker run --rm --volume="$PWD:/srv/jekyll" --volume="$PWD/vendor/bundle:/usr/ and open http://localhost:4000/rsd-engineeringcourse (or the link provided). Note that this is mounting the `bundle` directory where `act` will create them. + # Migration from jupyter notebooks to py:percent Using `jupytext` we've converted all the jupyter notebooks into plain text python files (py:percent) with: diff --git a/ch04packaging/03Packaging.ipynb.py b/ch04packaging/03Packaging.ipynb.py index a0b97411..60dbd0a2 100644 --- a/ch04packaging/03Packaging.ipynb.py +++ b/ch04packaging/03Packaging.ipynb.py @@ -21,23 +21,58 @@ # A good cross-platform build tool is the most important thing: you can always # have collaborators build from source. # +# Along with the content below, the Python ecosystem has a few fantastic guides on both simple +# and complex packaging. Some of the recommended ones are listed below - +# - [Python Packaging User Guide](https://packaging.python.org/en/latest/) +# - [pyOpenSci Python Package Guide](https://www.pyopensci.org/python-package-guide/) +# - [Scientific Python Packaging Guide]https://learn.scientific-python.org/development/tutorials/packaging/ +# +# Besides the guides, there are numerous "cookie" templates available for developers. These templates +# allow developers to generate an empty Python package following good practices and guidelines with a +# simple CLI command. +# - [Scientific Python cookie](https://github.com/scientific-python/cookie) +# - [Open Science Labs' scicookie](https://github.com/osl-incubator/scicookie) +# - Domain/ecosystem specific cookies like [pybamm-cookiecutter](https://github.com/pybamm-team/pybamm-cookiecutter) for battery modeling projects in Python exist too +# +# Finally, the packaging community regularly organises [PackagingCon](https://packaging-con.org) to discuss the packaging ecosystems +# of multiple languages and operating systems at a single place. +# # %% [markdown] # ### Distribution tools # %% [markdown] # Distribution tools allow one to obtain a working copy of someone else's package. -# -# - Language-specific tools: -# - python: PyPI, -# - ruby: Ruby Gems, -# - perl: CPAN, -# - R: CRAN +# The package managers are usually CLI utilities that allow you to query inside +# a repository of existing packages. +# +# - Language specific package/library managers: +# - python: [pip](https://pip.pypa.io), [pipx](https://pipx.pypa.io), [uv](https://github.com/astral-sh/uv), [conda](https://conda.org) +# - Julia: [Pkg](https://docs.julialang.org/en/v1/stdlib/Pkg/) +# - Rust: [Cargo](https://doc.rust-lang.org/cargo/) +# - R: using install.packages("package_name"), [devtools](https://cran.r-project.org/web/packages/devtools/index.html) +# - ruby: [gem](https://guides.rubygems.org/rubygems-basics/) +# - perl: [cpan](https://metacpan.org/dist/CPAN/view/scripts/cpan), [cpanm](https://metacpan.org/dist/App-cpanminus/view/bin/cpanm), [cpanp](https://metacpan.org/dist/CPANPLUS/view/bin/cpanp) # -# - Platform specific packagers e.g.: -# - [`brew`](https://brew.sh/) for MacOS, -# - `apt`/`dnf`/`pacman` for Linux or -# - [`choco`](https://chocolatey.org/) for Windows. +# - Platform specific package/library managers e.g.: +# - [`brew`](https://brew.sh/) for MacOS, +# - `apt`/`dnf`/`pacman` for Linux or +# - [`choco`](https://chocolatey.org/) for Windows. +# +# Every language has a repository or a central database of packages submitted +# by the developers. +# +# - Language-specific repositories: +# - python: [PyPI](https://pypi.org), [conda-forge](https://conda-forge.org) +# - Julia: [JuliaHub](https://juliahub.com/ui/Packages) +# - Rust: [Crates](https://crates.io) +# - R: [CRAN](https://www.cpan.org) +# - ruby: [RubyGems](https://rubygems.org) +# - perl: [CPAN](https://www.cpan.org) +# +# The difference between the package management tools and the package repositories is +# similar to the difference between Git and GitHub. +# # %% [markdown] # ### Laying out a project @@ -45,19 +80,20 @@ # %% [markdown] # # When planning to package a project for distribution, defining a suitable -# project layout is essential. A typical layout might look like this: +# project layout is essential. A typical scientific python compliant layout +# might look like this: # # ``` # repository_name -# |-- module_name -# | |-- __init__.py -# | |-- python_file.py -# | |-- another_python_file.py -# | `-- test -# | |-- fixtures -# | | `-- fixture_file.yaml -# | |-- __init__.py -# | `-- test_python_file.py +# |-- src +# | `-- package_name +# | |-- __init__.py # optional; required for exporting things under package's namespace +# | |-- python_file.py +# | |-- another_python_file.py +# `-- tests +# |-- fixtures +# | `-- fixture_file.yaml +# `-- test_python_file.py # |-- LICENSE.md # |-- CITATION.md # |-- README.md @@ -72,7 +108,8 @@ # To achieve this for our `greetings.py` file from the previous session, we can use the commands shown below. We can start by making our directory structure. You can create many nested directories at once using the `-p` switch on `mkdir`. # %% language="bash" -# mkdir -p greetings_repo/greetings/test/fixtures +# mkdir -p greetings_repo/src/greetings +# mkdir -p greetings_repo/tests/fixtures # %% [markdown] # For this notebook, since we are going to be modifying the files bit by bit, we are going to use the [autoreload ipython magic](https://ipython.readthedocs.io/en/stable/config/extensions/autoreload.html) so that we don't need to restart the kernel. @@ -94,30 +131,46 @@ # This information is included in the `project` section of our `pyproject.toml` file. # # But we also need to tell users how to build the package from these specifications. -# This information is specified in the `build-system` section of our `toml` file. -# In this case, we'll be using `setuptools` to build our package, so we list it in the `requires` field. -# We also need `setuptools_scm[toml]` so that `setuptools` can understand the settings we give it in our `.toml` file, and `wheel` to make the package distribution. -# -# Finally, we can set specific options for `setuptools` using additional sections in `pyproject.toml`: in this case, we will tell `setuptools` that it needs to find **and include** all of the files in our `greetings` folder. +# This information is specified in the `build-system` section of our `toml` file. Python packages are shipped on PyPI as "wheel" files, +# which are installable via `pip` and are generated by build backends. Wheels can be different for different platforms +# and different Python versions, and `pip` only resorts to installing a library through its source distribution (often referred as `SDist`) +# if it fails to find a compatible wheel file on PyPI. Python ecosystem houses a number of build backends, each of them spacializing in a different task. +# Some common Python build backends - +# - [`setuptools`](https://setuptools.pypa.io): allows building pure Python and Python + C/C++ projects +# - [`hatch`](https://hatch.pypa.io): allows building (and recommended for) pure Python projects +# - [`flit`](https://flit.pypa.io) allows building pure Python projects with minimal extra configurations +# - [`poetry`](https://python-poetry.org) allows building pure Python projects (a full blown dependency and environment management system) +# - [`scikit-build-core`](https://scikit-build-core.readthedocs.io) allows building pure Python and Python + C/C++ projects (under active development) +# - [`meson`](https://meson-python.readthedocs.io) allows building pure Python and Python + C/C++ projects (has a custom DSL) +# - [`maturin`](https://www.maturin.rs): allows building Rust binary extensions +# +# In this case, we'll be using `hatch` to build our package, so we list it in the `requires` field. Technically speaking, `hatch` is the front-end (a CLI utility) +# for the actual build-backend `hatchling`. `hatchling` is installed with hatch and can be specified as the `build-backend` in `pyproject.toml`. +# +# Finally, we can set specific options for `hatch` using additional sections in `pyproject.toml`: in this case, we will tell `hatch` that it needs to find **and include** all of the files in our `src` folder. +# The best way to look at all the options of a build-backend is by going through its documentation. # %% # %%writefile greetings_repo/pyproject.toml +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + [project] name = "Greetings" version = "0.1.0" -[build-system] -requires = ["setuptools", "setuptools_scm[toml]>=6.2", "wheel"] - -[tool.setuptools.packages.find] -include = ["greetings*"] - -# Add setuptools_scm if you need to generate version numbers from the git hash -#[tool.setuptools_scm] +[tool.hatch.build.targets.sdist] +include = [ + "src*", +] # %% [markdown] -# We can now install this "package" with pip: +# Some of the build-backends allow users to automate the package's version using VCS. +# For instance, you might want to look into [`hatch-vcs`](https://github.com/ofek/hatch-vcs) to enable VCS versioning with `hatch`. +# +# We can now install this "package" with pip (make sure `hatch` is installed): # %% language="bash" # cd greetings_repo @@ -130,21 +183,16 @@ # %% [markdown] # -# To create a regular package, we needed to have `__init__.py` files on each subdirectory that we want to be able to import. This is, since version 3.3 and the introduction of [Implicit Namespaces Packages](https://www.python.org/dev/peps/pep-0420/), not needed anymore. However, if you want to use relative imports and `pytest`, then you [still need to have these files](https://github.com/pytest-dev/pytest/issues/1927). +# To create a regular package, we needed to have `__init__.py` files on each subdirectory that we want to be able to import. This is, since version 3.3 and the introduction of [Implicit Namespaces Packages](https://www.python.org/dev/peps/pep-0420/), not needed anymore. # # The `__init__.py` files can contain any initialisation code you want to run when the (sub)module is imported. # -# For this example, and because we are using relative imports in the tests, we are creating the needed `__init__.py` files. - -# %% language="bash" +# For this example, we don't need to create the `__init__.py` files. # -# touch greetings_repo/greetings/__init__.py - -# %% [markdown] # And we can copy the `greet` function from the [previous section](https://github-pages.ucl.ac.uk/rsd-engineeringcourse/ch04packaging/02Argparse.html) in the `greeter.py` file. # %% -# %%writefile greetings_repo/greetings/greeter.py +# %%writefile greetings_repo/src/greetings/greeter.py def greet(personal, family, title="", polite=False): greeting = "How do you do, " if polite else "Hey, " @@ -168,7 +216,8 @@ def greet(personal, family, title="", polite=False): # %% from greetings.greeter import greet -greet("Terry","Gilliam") + +greet("Terry", "Gilliam") # %% [markdown] @@ -182,7 +231,7 @@ def greet(personal, family, title="", polite=False): # We need to add docstrings to our functions, so people can know how to use them. # %% -# %%writefile greetings_repo/greetings/greeter.py +# %%writefile greetings_repo/src/greetings/greeter.py def greet(personal, family, title="", polite=False): """ Generate a greeting string for a person. @@ -242,7 +291,7 @@ def greet(personal, family, title="", polite=False): # Note how we are importing `greet` using [relative imports](https://www.python.org/dev/peps/pep-0328/), where `.greeter` means to look for a `greeter` module within the same directory. # %% -# %%writefile greetings_repo/greetings/command.py +# %%writefile greetings_repo/src/greetings/command.py from argparse import ArgumentParser @@ -259,8 +308,14 @@ def process(): arguments = parser.parse_args() - print(greet(arguments.personal, arguments.family, - arguments.title, arguments.polite)) + print( + greet( + arguments.personal, + arguments.family, + arguments.title, + arguments.polite + ) + ) if __name__ == "__main__": @@ -277,6 +332,10 @@ def process(): # %% # %%writefile greetings_repo/pyproject.toml +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + [project] name = "Greetings" version = "0.1.0" @@ -284,18 +343,14 @@ def process(): [project.scripts] greet = "greetings.command:process" -[build-system] -requires = ["setuptools", "setuptools_scm[toml]>=6.2", "wheel"] - -[tool.setuptools.packages.find] -include = ["greetings*"] - -# Add setuptools_scm if you need to generate version numbers from the git hash -#[tool.setuptools_scm] +[tool.hatch.build.targets.sdist] +include = [ + "greetings/", +] # %% language="bash" # cd greetings_repo -# pip install -e . +# pip install . # %% [markdown] # @@ -319,7 +374,7 @@ def process(): # Let's give some life to our output using ascii art # %% -# %%writefile greetings_repo/greetings/command.py +# %%writefile greetings_repo/src/greetings/command.py from argparse import ArgumentParser @@ -342,6 +397,7 @@ def process(): arguments.title, arguments.polite) print(art("cute face"), message) + if __name__ == "__main__": process() @@ -352,6 +408,10 @@ def process(): # %% # %%writefile greetings_repo/pyproject.toml +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + [project] name = "Greetings" version = "0.1.0" @@ -362,21 +422,18 @@ def process(): [project.scripts] greet = "greetings.command:process" -[build-system] -requires = ["setuptools", "setuptools_scm[toml]>=6.2", "wheel"] - -[tool.setuptools.packages.find] -include = ["greetings*"] +[tool.hatch.build.targets.sdist] +include = [ + "src/", +] -# Add setuptools_scm if you need to generate version numbers from the git hash -#[tool.setuptools_scm] # %% [markdown] # When installing the package now, pip will also install the dependencies automatically. # %% language="bash" # cd greetings_repo -# pip install -e . +# pip install . # %% language="bash" # greet Terry Gilliam @@ -386,7 +443,7 @@ def process(): # %% [markdown] # -# We could now submit "greeter" to PyPI for approval, so everyone could `pip install` it. +# We could now submit "greeter" to PyPI, so everyone could `pip install` it. # # However, when using git, we don't even need to do that: we can install directly from any git URL: # @@ -468,17 +525,8 @@ def process(): Portions of the material are taken from [Software Carpentry](http://software-carpentry.org/) # %% [markdown] -# You may well want to formalise this using the [codemeta.json](https://codemeta.github.io/) standard or the [citation file format](http://citation-file-format.github.io/) - these don't have wide adoption yet, but we recommend it. - -# %% [markdown] -# ### Define packages and executables +# You may well want to formalise this using the [codemeta.json](https://codemeta.github.io/) standard or the [citation file format](http://citation-file-format.github.io/). -# %% [markdown] -# We need to create `__init__` files for the source and the tests. -# ```bash -# touch greetings/greetings/test/__init__.py -# touch greetings/greetings/__init__.py -# ``` # %% [markdown] # ### Write some unit tests @@ -486,13 +534,6 @@ def process(): # %% [markdown] # We can now write some tests to our library. # -# Remember, that we need to create the empty `__init__.py` files so that `pytest` can follow the relative imports. - -# %% language="bash" -# touch greetings_repo/greetings/test/__init__.py - -# %% [markdown] -# # Separating the script from the logical module made this possible. # # @@ -502,18 +543,22 @@ def process(): # # %% -# %%writefile greetings_repo/greetings/test/test_greeter.py +# %%writefile greetings_repo/tests/test_greeter.py import os import yaml -from ..greeter import greet +from greetings.greeter import greet def test_greet(): - with open(os.path.join(os.path.dirname(__file__), - 'fixtures', - 'samples.yaml')) as fixtures_file: + with open( + os.path.join( + os.path.dirname(__file__), + 'fixtures', + 'samples.yaml' + ) + ) as fixtures_file: fixtures = yaml.safe_load(fixtures_file) for fixture in fixtures: answer = fixture.pop('answer') @@ -533,7 +578,7 @@ def test_greet(): # # %% -# %%writefile greetings_repo/greetings/test/fixtures/samples.yaml +# %%writefile greetings_repo/tests/fixtures/samples.yaml - personal: Eric family: Idle @@ -559,19 +604,23 @@ def test_greet(): # However, this hasn't told us that also the third test is wrong too! A better aproach is to parametrize the testfile `greetings_repo/greetings/test/test_greeter.py` as follows: # %% -# %%writefile greetings_repo/greetings/test/test_greeter.py +# %%writefile greetings_repo/tests/test_greeter.py import os import pytest import yaml -from ..greeter import greet +from greetings.greeter import greet def read_fixture(): - with open(os.path.join(os.path.dirname(__file__), - 'fixtures', - 'samples.yaml')) as fixtures_file: + with open( + os.path.join( + os.path.dirname(__file__), + 'fixtures', + 'samples.yaml' + ) + ) as fixtures_file: fixtures = yaml.safe_load(fixtures_file) return fixtures @@ -599,7 +648,7 @@ def test_greeter(fixture): # %% [markdown] # Finally, we typically don't want to include the tests when we distribute our software for our users. -# We can make sure they are not included using the `exclude` option on when telling `setuptools` to find packages. +# We can also add pytest as an "optional" dependency for the developers of our package. # # Additionally, we can make sure that our README and LICENSE are included in our package metadata by declaring them in the `readme` and `license` fields under the `project` section. # If you're using a particularly common or standard license, you can even provide the name of the license, rather than the file, and your package builder will take care of the rest! @@ -607,11 +656,15 @@ def test_greeter(fixture): # %% # %%writefile greetings_repo/pyproject.toml +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + [project] name = "Greetings" version = "0.1.0" -readme = "README.md" -license = { file = "LICENSE.md" } +readme = { file = "README.md", content-type = "text/markdown" } +license-files = { paths = ["LICENSE.md"] } dependencies = [ "art", "pyyaml", @@ -620,15 +673,12 @@ def test_greeter(fixture): [project.scripts] greet = "greetings.command:process" -[build-system] -requires = ["setuptools", "setuptools_scm[toml]>=6.2", "wheel"] +[project.optional-dependencies] +dev = ["pytest >= 6"] -[tool.setuptools.packages.find] -include = ["greetings*"] -exclude = ["tests*"] +[tool.hatch.build.targets.sdist] +include = ["src*"] -# Add setuptools_scm if you need to generate version numbers from the git hash -#[tool.setuptools_scm] # %% [markdown] # ### Developer Install @@ -651,6 +701,12 @@ def test_greeter(fixture): # ```bash # pip install -e . # ``` +# +# with installing the `dev` dependencies: +# +# ```bash +# pip install -e ".[dev]" +# ``` # %% [markdown] # ### Distributing compiled code