Skip to content

Commit 661f814

Browse files
dpkpdpkp
authored
Fix GHA test workflow (#2456)
* Drop dist packaging steps and focus instead on testing. * Merge kafka/java matrix. * Separate pylint step, ignore errors for now.
1 parent 0ab54b9 commit 661f814

File tree

3 files changed

+44
-157
lines changed

3 files changed

+44
-157
lines changed

‎.github/workflows/python-package.yml‎

Lines changed: 42 additions & 155 deletions
Original file line numberDiff line numberDiff line change
@@ -1,198 +1,85 @@
1-
name: CI/CD
1+
# Derived from https://github.com/actions/starter-workflows/blob/main/ci/python-package.yml
2+
#
3+
name: Python Package
24

35
on:
46
push:
57
branches: ["master"]
68
pull_request:
79
branches: ["master"]
8-
release:
9-
types: [created]
10-
branches:
11-
- 'master'
12-
workflow_dispatch:
1310

1411
env:
1512
FORCE_COLOR: "1" # Make tools pretty.
1613
PIP_DISABLE_PIP_VERSION_CHECK: "1"
1714
PIP_NO_PYTHON_VERSION_WARNING: "1"
18-
PYTHON_LATEST: "3.12"
19-
KAFKA_LATEST: "2.6.0"
20-
21-
# For re-actors/checkout-python-sdist
22-
sdist-artifact: python-package-distributions
2315

2416
jobs:
17+
build:
2518

26-
build-sdist:
27-
name: 📦 Build the source distribution
28-
runs-on: ubuntu-latest
29-
steps:
30-
- name: Checkout project
31-
uses: actions/checkout@v4
32-
with:
33-
fetch-depth: 0
34-
- name: Set up Python
35-
uses: actions/setup-python@v5
36-
with:
37-
python-version: ${{ env.PYTHON_LATEST }}
38-
cache: pip
39-
- run: python -m pip install build
40-
name: Install core libraries for build and install
41-
- name: Build artifacts
42-
run: python -m build
43-
- name: Upload built artifacts for testing
44-
uses: actions/upload-artifact@v3
45-
with:
46-
name: ${{ env.sdist-artifact }}
47-
# NOTE: Exact expected file names are specified here
48-
# NOTE: as a safety measure — if anything weird ends
49-
# NOTE: up being in this dir or not all dists will be
50-
# NOTE: produced, this will fail the workflow.
51-
path: dist/${{ env.sdist-name }}
52-
retention-days: 15
53-
54-
test-python:
55-
name: Tests on ${{ matrix.python-version }}
56-
needs:
57-
- build-sdist
5819
runs-on: ubuntu-latest
59-
continue-on-error: ${{ matrix.experimental }}
20+
name: "Test: python ${{ matrix.python }} / kafka ${{ matrix.kafka }}"
21+
continue-on-error: ${{ matrix.experimental || false }}
6022
strategy:
6123
fail-fast: false
6224
matrix:
63-
python-version:
64-
- "3.8"
65-
- "3.9"
66-
- "3.10"
67-
- "3.11"
25+
kafka:
26+
- "0.8.2.2"
27+
- "0.9.0.1"
28+
- "0.10.2.2"
29+
- "0.11.0.3"
30+
- "1.1.1"
31+
- "2.4.0"
32+
- "2.5.0"
33+
- "2.6.0"
34+
python:
6835
- "3.12"
69-
experimental: [ false ]
7036
include:
71-
- python-version: "pypy3.9"
72-
experimental: true
73-
- python-version: "~3.13.0-0"
74-
experimental: true
37+
#- python: "pypy3.9"
38+
# kafka: "2.6.0"
39+
# experimental: true
40+
#- python: "~3.13.0-0"
41+
# kafka: "2.6.0"
42+
# experimental: true
43+
- python: "3.8"
44+
kafka: "2.6.0"
45+
- python: "3.9"
46+
kafka: "2.6.0"
47+
- python: "3.10"
48+
kafka: "2.6.0"
49+
- python: "3.11"
50+
kafka: "2.6.0"
51+
7552
steps:
76-
- name: Checkout the source code
77-
uses: actions/checkout@v4
78-
with:
79-
fetch-depth: 0
80-
- name: Setup java
81-
uses: actions/setup-java@v4
82-
with:
83-
distribution: temurin
84-
java-version: 11
85-
- name: Set up Python
53+
- uses: actions/checkout@v4
54+
- name: Set up Python ${{ matrix.python }}
8655
uses: actions/setup-python@v5
8756
with:
88-
python-version: ${{ matrix.python-version }}
57+
python-version: ${{ matrix.python }}
8958
cache: pip
9059
cache-dependency-path: |
9160
requirements-dev.txt
92-
- name: Check Java installation
93-
run: source travis_java_install.sh
94-
- name: Pull Kafka releases
95-
run: ./build_integration.sh
96-
env:
97-
PLATFORM: ${{ matrix.platform }}
98-
KAFKA_VERSION: ${{ env.KAFKA_LATEST }}
99-
# TODO: Cache releases to expedite testing
10061
- name: Install dependencies
10162
run: |
10263
sudo apt install -y libsnappy-dev libzstd-dev
10364
python -m pip install --upgrade pip
104-
python -m pip install tox tox-gh-actions
105-
pip install .
10665
pip install -r requirements-dev.txt
107-
- name: Test with tox
108-
run: tox
109-
env:
110-
PLATFORM: ${{ matrix.platform }}
111-
KAFKA_VERSION: ${{ env.KAFKA_LATEST }}
112-
113-
test-kafka:
114-
name: Tests for Kafka ${{ matrix.kafka-version }}
115-
needs:
116-
- build-sdist
117-
runs-on: ubuntu-latest
118-
strategy:
119-
fail-fast: false
120-
matrix:
121-
kafka-version:
122-
- "0.8.2.2"
123-
- "0.9.0.1"
124-
- "0.10.2.2"
125-
- "0.11.0.2"
126-
- "0.11.0.3"
127-
- "1.1.1"
128-
- "2.4.0"
129-
- "2.5.0"
130-
- "2.6.0"
131-
steps:
132-
- name: Checkout the source code
133-
uses: actions/checkout@v4
134-
with:
135-
fetch-depth: 0
66+
pip install tox-gh-actions
67+
- name: Pylint
68+
run: pylint --recursive=y --errors-only --exit-zero kafka test
13669
- name: Setup java
13770
uses: actions/setup-java@v4
13871
with:
13972
distribution: temurin
140-
java-version: 8
141-
- name: Set up Python
142-
uses: actions/setup-python@v5
143-
with:
144-
python-version: ${{ env.PYTHON_LATEST }}
145-
cache: pip
146-
cache-dependency-path: |
147-
requirements-dev.txt
73+
java-version: 11
74+
- name: Check Java installation
75+
run: source travis_java_install.sh
14876
- name: Pull Kafka releases
14977
run: ./build_integration.sh
15078
env:
151-
# This is fast enough as long as you pull only one release at a time,
152-
# no need to worry about caching
15379
PLATFORM: ${{ matrix.platform }}
154-
KAFKA_VERSION: ${{ matrix.kafka-version }}
155-
- name: Install dependencies
156-
run: |
157-
sudo apt install -y libsnappy-dev libzstd-dev
158-
python -m pip install --upgrade pip
159-
python -m pip install tox tox-gh-actions
160-
pip install .
161-
pip install -r requirements-dev.txt
80+
KAFKA_VERSION: ${{ matrix.kafka }}
16281
- name: Test with tox
16382
run: tox
16483
env:
16584
PLATFORM: ${{ matrix.platform }}
166-
KAFKA_VERSION: ${{ matrix.kafka-version }}
167-
168-
check: # This job does nothing and is only used for the branch protection
169-
name: ✅ Ensure the required checks passing
170-
if: always()
171-
needs:
172-
- build-sdist
173-
- test-python
174-
- test-kafka
175-
runs-on: ubuntu-latest
176-
steps:
177-
- name: Decide whether the needed jobs succeeded or failed
178-
uses: re-actors/alls-green@release/v1
179-
with:
180-
jobs: ${{ toJSON(needs) }}
181-
publish:
182-
name: 📦 Publish to PyPI
183-
runs-on: ubuntu-latest
184-
needs: [build-sdist]
185-
permissions:
186-
id-token: write
187-
environment: pypi
188-
if: github.event_name == 'release' && github.event.action == 'created'
189-
steps:
190-
- name: Download the sdist artifact
191-
uses: actions/download-artifact@v3
192-
with:
193-
name: artifact
194-
path: dist
195-
- name: Publish package to PyPI
196-
uses: pypa/gh-action-pypi-publish@release/v1
197-
with:
198-
password: ${{ secrets.PYPI_API_TOKEN }}
85+
KAFKA_VERSION: ${{ matrix.kafka }}

‎build_integration.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ pushd servers
4848
echo "Downloading kafka ${kafka} tarball"
4949
TARBALL=${DIST_BASE_URL}${kafka}/${KAFKA_ARTIFACT}
5050
if command -v wget 2>/dev/null; then
51-
wget -N $TARBALL
51+
wget -nv -N $TARBALL
5252
else
5353
echo "wget not found... using curl"
5454
curl -f $TARBALL -o ${KAFKA_ARTIFACT}

‎tox.ini‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ deps =
2929
xxhash
3030
crc32c
3131
commands =
32-
pytest {posargs:--pylint --pylint-rcfile=pylint.rc --pylint-error-types=EF --cov=kafka --cov-config=.covrc}
32+
pytest {posargs:--cov=kafka --cov-config=.covrc}
3333
setenv =
3434
CRC32C_SW_MODE = auto
3535
PROJECT_ROOT = {toxinidir}

0 commit comments

Comments
 (0)