diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 669e956..5fba4cb 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -56,7 +56,12 @@ jobs: uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 with: disable-sudo: true - egress-policy: audit + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + objects.githubusercontent.com:443 + uploads.github.com:443 - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 5a1e406..12ef219 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -24,6 +24,7 @@ jobs: allowed-endpoints: > api.deps.dev:443 api.github.com:443 + api.securityscorecards.dev:443 github.com:443 - name: Checkout Repository diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4f484db..538e912 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,7 +33,12 @@ jobs: - name: Harden Runner uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 with: - egress-policy: audit + disable-sudo: true + egress-policy: block + allowed-endpoints: > + files.pythonhosted.org:443 + github.com:443 + pypi.org:443 - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -72,7 +77,16 @@ jobs: - name: Harden Runner uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 with: - egress-policy: audit + disable-sudo: true + egress-policy: block + allowed-endpoints: > + coveralls.io:443 + docs.scipy.org:443 + files.pythonhosted.org:443 + github.com:443 + pandas.pydata.org:443 + pypi.org:443 + raw.githubusercontent.com:443 - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -114,7 +128,17 @@ jobs: - name: Harden Runner uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 with: - egress-policy: audit + disable-sudo: true + egress-policy: block + allowed-endpoints: > + conda.anaconda.org:443 + coveralls.io:443 + files.pythonhosted.org:443 + github.com:443 + objects.githubusercontent.com:443 + pypi.org:443 + raw.githubusercontent.com:443 + repo.anaconda.com:443 - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -127,10 +151,6 @@ jobs: environment-file: environment-dev.yml create-args: >- python=${{ matrix.python-version }} -# - name: Install xclim development version -# run: | -# echo "Installing xclim from main branch until version 0.53.0+ is released" -# python -m pip install git+https://github.com/Ouranosinc/xclim.git@main - name: Install xsdba run: | python -m pip install --no-deps . @@ -160,7 +180,11 @@ jobs: uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 with: disable-sudo: true - egress-policy: audit + egress-policy: block + allowed-endpoints: > + coveralls.io:443 + github.com:443 + objects.githubusercontent.com:443 - name: Coveralls Finished uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b # v2.3.6 with: diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml new file mode 100644 index 0000000..19b9f81 --- /dev/null +++ b/.github/workflows/upstream.yml @@ -0,0 +1,112 @@ +name: Test Upstream Dependencies +on: + push: + branches: + - main + paths-ignore: + - CHANGELOG.rst + - README.rst + - pyproject.toml + - src/xsdba/__init__.py + schedule: + - cron: "0 0 * * *" # Daily “At 00:00” UTC + workflow_dispatch: # allows you to trigger the workflow run manually + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + upstream-dev: + name: test-upstream-dev (Python${{ matrix.python-version }}) + runs-on: ubuntu-latest + permissions: + issues: write + if: | + (github.event_name == 'schedule') || + (github.event_name == 'workflow_dispatch') || + (github.event_name == 'push') + strategy: + fail-fast: false + matrix: + python-version: [ "3.12" ] + testdata-cache: [ '~/.cache/xclim-testdata' ] + defaults: + run: + shell: bash -l {0} + steps: + - name: Harden Runner + uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 + with: + disable-sudo: true + egress-policy: audit + - name: Start Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@e79d33b08b0c47de4438c161d432730fce23f246 # v4.2.0 + with: + task: start-measurement + branch: ${{ github.head_ref || github.ref_name }} + - name: Checkout Repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }} + uses: mamba-org/setup-micromamba@0dea6379afdaffa5d528b3d1dabc45da37f443fc # v2.0.4 + with: + cache-downloads: true + cache-environment: true + environment-file: environment.yml + create-args: >- + pytest-reportlog + python=${{ matrix.python-version }} + - name: Install upstream versions + run: | + # git-based dependencies cannot be installed from hashes + python -m pip install -r CI/requirements_upstream.txt + - name: Install xsdba + run: | + python -m pip install --no-user --no-deps --editable . + - name: Check versions + run: | + micromamba list + python -m pip check || true + - name: Setup Python Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@e79d33b08b0c47de4438c161d432730fce23f246 # v4.2.0 + with: + task: get-measurement + label: 'Environment Setup (Upstream, Python${{ matrix.python-version }})' + continue-on-error: true + - name: Test Data Caching + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: | + ${{ matrix.testdata-cache }} + key: ${{ runner.os }}-xclim-testdata-upstream-${{ hashFiles('pyproject.toml', 'tox.ini') }} + - name: Run Tests + if: success() + id: status + run: | + python -m pytest --numprocesses=logical --durations=10 --cov=xsdba --cov-report=term-missing --report-log output-${{ matrix.python-version }}-log.jsonl + - name: Generate and publish the report + if: | + failure() + && steps.status.outcome == 'failure' + && github.event_name == 'schedule' + && github.repository_owner == 'Ouranosinc' + uses: xarray-contrib/issue-from-pytest-log@f94477e45ef40e4403d7585ba639a9a3bcc53d43 # v1.3.0 + with: + issue-title: "⚠️ Nightly upstream-dev CI failed for Python${{ matrix.python-version }} ⚠️" + log-path: output-${{ matrix.python-version }}-log.jsonl + - name: Tests measurement + uses: green-coding-solutions/eco-ci-energy-estimation@e79d33b08b0c47de4438c161d432730fce23f246 # v4.2.0 + with: + task: get-measurement + label: 'Testing and Reporting (Upstream, Python${{ matrix.python-version }})' + continue-on-error: true + - name: Show Energy Results + uses: green-coding-solutions/eco-ci-energy-estimation@e79d33b08b0c47de4438c161d432730fce23f246 # v4.2.0 + with: + task: display-results + continue-on-error: true diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 17cdbd5..4f6a59b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,7 @@ Changes * Now using advanced CodeQL configuration. * New pre-commit hooks for `vulture` (find dead code), `codespell` (grammatical errors), `zizmor` (workflow security), and `gitleaks` (token commit prevention). * Corrected some minor spelling and security issues. +* Added `upstream` testing to the CI pipeline for both daily and push events. (:pull:`61`). Fixes ^^^^^ @@ -36,7 +37,7 @@ Changes * Split `sdba` from `xclim` into its own standalone package. Where needed, some common functionalities were duplicated: (:pull:`8`) * ``xsdba.units`` is an adaptation of the ``xclim.core.units`` modules. * Many functions and definitions found in ``xclim.core.calendar`` have been adapted to ``xsdba.base``. -* Dependencies have been updated to reflect the new package structure. (:pull:`45`) +* Dependencies have been updated to reflect the new package structure. (:pull:`45`). * Updated documentation configuration: (:pull:`46`) * Significant improvements to the documentation content and layout. * Now using the `furo` theme for `sphinx`. diff --git a/CI/requirements_upstream.txt b/CI/requirements_upstream.txt new file mode 100644 index 0000000..06f6780 --- /dev/null +++ b/CI/requirements_upstream.txt @@ -0,0 +1,2 @@ +xarray @ git+https://github.com/pydata/xarray.git@main +xclim @ git+https://github.com/Ouranosinc/xclim.git@main diff --git a/tox.ini b/tox.ini index aa91405..638c870 100644 --- a/tox.ini +++ b/tox.ini @@ -53,6 +53,7 @@ download = True install_command = python -m pip install --no-user {opts} {packages} deps = coveralls: coveralls>=4.0.1 + upstream: -r {toxinidir}/CI/requirements_upstream.txt commands_pre = pip list pip check