From a5e089fb590b0d9c61ecd12a4164df76f6e67a58 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 21 Jan 2025 09:55:00 -0500 Subject: [PATCH 1/5] add upstream testing Signed-off-by: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> --- .github/workflows/upstream.yml | 130 +++++++++++++++++++++++++++++++++ CHANGELOG.rst | 4 +- CI/requirements_upstream.txt | 1 + tox.ini | 4 +- 4 files changed, 134 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/upstream.yml create mode 100644 CI/requirements_upstream.txt diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml new file mode 100644 index 0000000..4d0d264 --- /dev/null +++ b/.github/workflows/upstream.yml @@ -0,0 +1,130 @@ +name: Test Upstream Dependencies +on: + push: + branches: + - main + paths-ignore: + - CHANGELOG.rst + - README.rst + - pyproject.toml + - src/xsdba/__init__.py + schedule: + - cron: "0 0 * * *" # Daily “At 00:00” UTC + workflow_dispatch: # allows you to trigger the workflow run manually + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + upstream-dev: + name: test-upstream-dev (Python${{ matrix.python-version }}) + runs-on: ubuntu-latest + permissions: + issues: write + if: | + (github.event_name == 'schedule') || + (github.event_name == 'workflow_dispatch') || + (github.event_name == 'push') + strategy: + fail-fast: false + matrix: + python-version: [ "3.12" ] + testdata-cache: [ '~/.cache/xclim-testdata' ] + defaults: + run: + shell: bash -l {0} + steps: + - name: Harden Runner + uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.electricitymap.org:443 + api.github.com:443 + api.green-coding.io:443 + api.securityscorecards.dev:443 + conda.anaconda.org:443 + dap.service.does.not.exist:443 + files.pythonhosted.org:443 + github.com:443 + ip-api.com:80 + ipapi.co:443 + objects.githubusercontent.com:443 + proxy.golang.org:4433 + pypi.org:443 + raw.githubusercontent.com:443 + repo.anaconda.com:443 + sum.golang.org:443 + - name: Start Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@e79d33b08b0c47de4438c161d432730fce23f246 # v4.2.0 + with: + task: start-measurement + branch: ${{ github.head_ref || github.ref_name }} + - name: Checkout Repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }} + uses: mamba-org/setup-micromamba@0dea6379afdaffa5d528b3d1dabc45da37f443fc # v2.0.4 + with: + cache-downloads: true + cache-environment: true + environment-file: environment.yml + create-args: >- + pytest-reportlog + python=${{ matrix.python-version }} + - name: Install upstream versions + run: | + # git-based dependencies cannot be installed from hashes + python -m pip install -r CI/requirements_upstream.txt + - name: Install xsdba + run: | + python -m pip install --no-user --no-deps --editable . + - name: Check versions + run: | + micromamba list + xclim show_version_info + python -m pip check || true + - name: Setup Python Measurement + uses: green-coding-solutions/eco-ci-energy-estimation@e79d33b08b0c47de4438c161d432730fce23f246 # v4.2.0 + with: + task: get-measurement + label: 'Environment Setup (Upstream, Python${{ matrix.python-version }})' + continue-on-error: true + - name: Test Data Caching + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: | + ${{ matrix.testdata-cache }} + key: ${{ runner.os }}-xclim-testdata-upstream-${{ hashFiles('pyproject.toml', 'tox.ini') }} + - name: Run Tests + if: success() + id: status + run: | + python -m pytest --numprocesses=logical --durations=10 --cov=xclim --cov-report=term-missing --report-log output-${{ matrix.python-version }}-log.jsonl + - name: Generate and publish the report + if: | + failure() + && steps.status.outcome == 'failure' + && github.event_name == 'schedule' + && github.repository_owner == 'Ouranosinc' + uses: xarray-contrib/issue-from-pytest-log@f94477e45ef40e4403d7585ba639a9a3bcc53d43 # v1.3.0 + with: + issue-title: "⚠️ Nightly upstream-dev CI failed for Python${{ matrix.python-version }} ⚠️" + log-path: output-${{ matrix.python-version }}-log.jsonl + - name: Tests measurement + uses: green-coding-solutions/eco-ci-energy-estimation@e79d33b08b0c47de4438c161d432730fce23f246 # v4.2.0 + with: + task: get-measurement + label: 'Testing and Reporting (Upstream, Python${{ matrix.python-version }})' + continue-on-error: true + - name: Show Energy Results + uses: green-coding-solutions/eco-ci-energy-estimation@e79d33b08b0c47de4438c161d432730fce23f246 # v4.2.0 + with: + task: display-results + continue-on-error: true diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d00f55a..c88b059 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,11 +5,11 @@ Changelog `Unreleased `_ (latest) ------------------------------------------------------------ -Contributors: Pascal Bourgault (:user:`aulemahal`), Éric Dupuis (:user:`coxipi`). +Contributors: Pascal Bourgault (:user:`aulemahal`), Éric Dupuis (:user:`coxipi`), Trevor James Smith (:user:`Zeitsperre`). Changes ^^^^^^^ -* No change. +* Added daily `upstream` testing to the CI pipeline. Fixes ^^^^^ diff --git a/CI/requirements_upstream.txt b/CI/requirements_upstream.txt new file mode 100644 index 0000000..1f66e39 --- /dev/null +++ b/CI/requirements_upstream.txt @@ -0,0 +1 @@ +xarray @ git+https://github.com/pydata/xarray.git@main diff --git a/tox.ini b/tox.ini index 6cd981e..74e1efa 100644 --- a/tox.ini +++ b/tox.ini @@ -52,9 +52,7 @@ extras = download = True install_command = python -m pip install --no-user {opts} {packages} deps = -; If you want to make tox run the tests with the same versions, create a -; requirements.txt with the pinned versions and uncomment the following line: -; -r{toxinidir}/requirements.txt + upstream: -r {toxinidir}/CI/requirements_upstream.txt commands_pre = pip list pip check From d30cd58305fac0370d623bddd0f3382b265010ea Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 21 Jan 2025 09:57:20 -0500 Subject: [PATCH 2/5] update CHANGELOG.rst Signed-off-by: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> --- CHANGELOG.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c88b059..44d86bd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,12 +9,12 @@ Contributors: Pascal Bourgault (:user:`aulemahal`), Éric Dupuis (:user:`coxipi` Changes ^^^^^^^ -* Added daily `upstream` testing to the CI pipeline. +* Added daily `upstream` testing to the CI pipeline. (:pull:`61`). Fixes ^^^^^ -* Gave credits to the package to all previous contributors of ``xclim.sdba`` (:issue:`58`, :pull:`59`). -* Pin `sphinx-codeautolink` to fix ReadTheDocs and correct some docs errors (:pull:`40`). +* Gave credits to the package to all previous contributors of ``xclim.sdba``. (:issue:`58`, :pull:`59`). +* Pin `sphinx-codeautolink` to fix ReadTheDocs and correct some docs errors. (:pull:`40`). .. _changes_0.2.0: @@ -28,7 +28,7 @@ Changes * Split `sdba` from `xclim` into its own standalone package. Where needed, some common functionalities were duplicated: (:pull:`8`) * ``xsdba.units`` is an adaptation of the ``xclim.core.units`` modules. * Many functions and definitions found in ``xclim.core.calendar`` have been adapted to ``xsdba.base``. -* Dependencies have been updated to reflect the new package structure. (:pull:`45`) +* Dependencies have been updated to reflect the new package structure. (:pull:`45`). * Updated documentation configuration: (:pull:`46`) * Significant improvements to the documentation content and layout. * Now using the `furo` theme for `sphinx`. From 94fcaa832ffc78c80cc7b00a157bc9241d8ac1c3 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Mon, 10 Feb 2025 11:07:42 -0500 Subject: [PATCH 3/5] Update .github/workflows/upstream.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Éric Dupuis <71575674+coxipi@users.noreply.github.com> --- .github/workflows/upstream.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index 4d0d264..5e2cdc2 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -106,7 +106,7 @@ jobs: if: success() id: status run: | - python -m pytest --numprocesses=logical --durations=10 --cov=xclim --cov-report=term-missing --report-log output-${{ matrix.python-version }}-log.jsonl + python -m pytest --numprocesses=logical --durations=10 --cov=xsdba --cov-report=term-missing --report-log output-${{ matrix.python-version }}-log.jsonl - name: Generate and publish the report if: | failure() From a13136ae5944233d5bb052d48fd46557bf66a997 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 18 Feb 2025 10:14:03 -0500 Subject: [PATCH 4/5] adjust security allowances --- .github/workflows/codeql.yml | 7 ++++- .github/workflows/dependency-review.yml | 1 + .github/workflows/main.yml | 40 ++++++++++++++++++++----- .github/workflows/upstream.yml | 20 +------------ 4 files changed, 40 insertions(+), 28 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 669e956..5fba4cb 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -56,7 +56,12 @@ jobs: uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 with: disable-sudo: true - egress-policy: audit + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + objects.githubusercontent.com:443 + uploads.github.com:443 - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 5a1e406..12ef219 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -24,6 +24,7 @@ jobs: allowed-endpoints: > api.deps.dev:443 api.github.com:443 + api.securityscorecards.dev:443 github.com:443 - name: Checkout Repository diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4f484db..538e912 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,7 +33,12 @@ jobs: - name: Harden Runner uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 with: - egress-policy: audit + disable-sudo: true + egress-policy: block + allowed-endpoints: > + files.pythonhosted.org:443 + github.com:443 + pypi.org:443 - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -72,7 +77,16 @@ jobs: - name: Harden Runner uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 with: - egress-policy: audit + disable-sudo: true + egress-policy: block + allowed-endpoints: > + coveralls.io:443 + docs.scipy.org:443 + files.pythonhosted.org:443 + github.com:443 + pandas.pydata.org:443 + pypi.org:443 + raw.githubusercontent.com:443 - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -114,7 +128,17 @@ jobs: - name: Harden Runner uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 with: - egress-policy: audit + disable-sudo: true + egress-policy: block + allowed-endpoints: > + conda.anaconda.org:443 + coveralls.io:443 + files.pythonhosted.org:443 + github.com:443 + objects.githubusercontent.com:443 + pypi.org:443 + raw.githubusercontent.com:443 + repo.anaconda.com:443 - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -127,10 +151,6 @@ jobs: environment-file: environment-dev.yml create-args: >- python=${{ matrix.python-version }} -# - name: Install xclim development version -# run: | -# echo "Installing xclim from main branch until version 0.53.0+ is released" -# python -m pip install git+https://github.com/Ouranosinc/xclim.git@main - name: Install xsdba run: | python -m pip install --no-deps . @@ -160,7 +180,11 @@ jobs: uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 with: disable-sudo: true - egress-policy: audit + egress-policy: block + allowed-endpoints: > + coveralls.io:443 + github.com:443 + objects.githubusercontent.com:443 - name: Coveralls Finished uses: coverallsapp/github-action@648a8eb78e6d50909eff900e4ec85cab4524a45b # v2.3.6 with: diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index 5e2cdc2..19b9f81 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -42,24 +42,7 @@ jobs: uses: step-security/harden-runner@cb605e52c26070c328afc4562f0b4ada7618a84e # v2.10.4 with: disable-sudo: true - egress-policy: block - allowed-endpoints: > - api.electricitymap.org:443 - api.github.com:443 - api.green-coding.io:443 - api.securityscorecards.dev:443 - conda.anaconda.org:443 - dap.service.does.not.exist:443 - files.pythonhosted.org:443 - github.com:443 - ip-api.com:80 - ipapi.co:443 - objects.githubusercontent.com:443 - proxy.golang.org:4433 - pypi.org:443 - raw.githubusercontent.com:443 - repo.anaconda.com:443 - sum.golang.org:443 + egress-policy: audit - name: Start Measurement uses: green-coding-solutions/eco-ci-energy-estimation@e79d33b08b0c47de4438c161d432730fce23f246 # v4.2.0 with: @@ -88,7 +71,6 @@ jobs: - name: Check versions run: | micromamba list - xclim show_version_info python -m pip check || true - name: Setup Python Measurement uses: green-coding-solutions/eco-ci-energy-estimation@e79d33b08b0c47de4438c161d432730fce23f246 # v4.2.0 From d030d0daab371c913e76f025746423d2d701bd46 Mon Sep 17 00:00:00 2001 From: Trevor James Smith <10819524+Zeitsperre@users.noreply.github.com> Date: Tue, 18 Feb 2025 10:14:14 -0500 Subject: [PATCH 5/5] add xclim@main --- CI/requirements_upstream.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CI/requirements_upstream.txt b/CI/requirements_upstream.txt index 1f66e39..06f6780 100644 --- a/CI/requirements_upstream.txt +++ b/CI/requirements_upstream.txt @@ -1 +1,2 @@ xarray @ git+https://github.com/pydata/xarray.git@main +xclim @ git+https://github.com/Ouranosinc/xclim.git@main