-
-
Notifications
You must be signed in to change notification settings - Fork 292
339 lines (330 loc) · 13.2 KB
/
ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
name: CI
on: [push, pull_request]
defaults:
run:
shell: bash
env:
_PEX_TEST_DEV_ROOT: ${{ github.workspace }}/.pex_dev
_PEX_TEST_POS_ARGS: "--color --devpi --devpi-timeout 15.0 --shutdown-devpi -vvs"
_PEX_PEXPECT_TIMEOUT: 10
# We have integration tests that exercise `--scie` support and these can trigger downloads from
# GitHub Releases that needed elevated rate limit quota, which this gives.
SCIENCE_AUTH_API_GITHUB_COM_BEARER: ${{ secrets.GITHUB_TOKEN }}
# We fetch Windows script executable stubs when building Pex.
_PEX_FETCH_WINDOWS_STUBS_BEARER: ${{ secrets.GITHUB_TOKEN }}
concurrency:
group: CI-${{ github.ref }}
# Queue on all branches and tags, but only cancel overlapping PR burns.
cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/') }}
jobs:
setup:
name: Check GitHub Organization
if: github.repository_owner == 'pex-tool'
runs-on: ubuntu-24.04
outputs:
pex_test_posargs: ${{ env._PEX_TEST_POS_ARGS }}
steps:
- name: Noop
if: false
run: |
echo "This is a dummy step that will never run."
checks:
name: "Unix: tox -e format-check,lint-check,typecheck,enum-check,vendor-check,package,docs"
needs: setup
runs-on: ubuntu-24.04
steps:
- name: Checkout Pex
uses: actions/checkout@v4
with:
# We need branches and tags since package leans on `git describe`. Passing 0 gets us
# complete history.
fetch-depth: 0
- name: Setup Python 3.9
uses: actions/setup-python@v5
with:
# We need to keep Python 3.9 for consistent vendoring with tox.
python-version: "3.9"
- name: Check Formatting, Lints and Types
uses: pex-tool/actions/run-tox@c53dadd8b410bbd66480de91067e9e45d2b3af38
with:
tox-env: format-check,lint-check,typecheck
- name: Check Enum Types
run: |
BASE_MODE=pull ./dtox.sh -e enum-check -- -v --require-py27
- name: Check Vendoring
uses: pex-tool/actions/run-tox@c53dadd8b410bbd66480de91067e9e45d2b3af38
with:
tox-env: vendor-check
- name: Check Packaging
uses: pex-tool/actions/run-tox@c53dadd8b410bbd66480de91067e9e45d2b3af38
with:
tox-env: >-
package -- --additional-format sdist --additional-format wheel --embed-docs --clean-docs
--scies --gen-md-table-of-hash-and-size dist/hashes.md
- name: Check Docs
uses: pex-tool/actions/run-tox@c53dadd8b410bbd66480de91067e9e45d2b3af38
with:
tox-env: docs -- --linkcheck --pdf --clean-html
# N.B.: The name of this job key (linux-tests) is depended on by scrips/build_cache_image.py. In
# particular, the tox-env matrix list is used to ensure the cache covers all Linux CI jobs.
linux-tests:
name: "Linux: ./dtox.sh -e ${{ matrix.tox-env }} ${{ matrix.pex-test-pos-args }}"
needs: setup
runs-on: ubuntu-24.04
strategy:
matrix:
include:
# N.B.: When editing these, also edit build-cache-image.yml.
# Unit tests:
# -----------
- tox-env: py27-pip20
- tox-env: py313-pip25_0_1
- tox-env: pypy310-pip24_3_1
# Integration tests, split most into two shards:
# ----------------------------------------------
# CPython 2.7 is fast enough not to require sharding.
- tox-env: py27-pip20-integration
- tox-env: py38-pip22_3_1-integration
pex-test-pos-args: --shard 1/2
- tox-env: py38-pip22_3_1-integration
pex-test-pos-args: --shard 2/2
- tox-env: py313-pip25_0_1-integration
pex-test-pos-args: --shard 1/2
- tox-env: py313-pip25_0_1-integration
pex-test-pos-args: --shard 2/2
- tox-env: py314-pip25_0_1-integration
pex-test-pos-args: --shard 1/2
- tox-env: py314-pip25_0_1-integration
pex-test-pos-args: --shard 2/2
- tox-env: pypy310-pip24_3_1-integration
pex-test-pos-args: --shard 1/2
- tox-env: pypy310-pip24_3_1-integration
pex-test-pos-args: --shard 2/2
steps:
- name: Free Up Disk Space
uses: jlumbroso/[email protected]
with:
android: true # ~14GB
dotnet: true # ~2GB
tool-cache: true # ~12GB
# Too little space savings or too slow.
haskell: false
large-packages: false
docker-images: false
swap-storage: false
- name: Checkout Pex
uses: actions/checkout@v4
with:
# We need branches and tags for some ITs.
fetch-depth: 0
# Some ITs need this for VCS URLs of the form git+ssh://[email protected]/...
- name: Setup SSH Agent
uses: webfactory/[email protected]
env:
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
if: env.SSH_PRIVATE_KEY != ''
with:
ssh-private-key: ${{ env.SSH_PRIVATE_KEY }}
- name: Run Tests
run: |
# This is needed to get pexpect tests working under PyPy running under docker.
export TERM="xterm"
BASE_MODE=pull CACHE_MODE=pull \
./dtox.sh -e ${{ matrix.tox-env }} -- \
${{ env._PEX_TEST_POS_ARGS }} ${{ matrix.pex-test-pos-args }}
mac-tests:
name: "Mac: tox -e ${{ matrix.tox-env }} ${{ matrix.pex-test-pos-args }}"
needs: setup
runs-on: macos-13
strategy:
matrix:
include:
- python-version: [ 3, 13 ]
tox-env: py313-pip25_0_1
cache: true
tox-env-python: python3.11
- python-version: [ 3, 13 ]
tox-env: py313-pip25_0_1-integration
cache: true
tox-env-python: python3.11
pex-test-pos-args: --shard 1/2
- python-version: [ 3, 13 ]
tox-env: py313-pip25_0_1-integration
cache: true
tox-env-python: python3.11
pex-test-pos-args: --shard 2/2
steps:
- name: Checkout Pex
uses: actions/checkout@v4
with:
# We need branches and tags for some ITs.
fetch-depth: 0
path: repo
- name: Setup Python ${{ join(matrix.python-version, '.') }}
uses: actions/setup-python@v5
with:
python-version: "${{ join(matrix.python-version, '.') }}"
- name: Expose Pythons
uses: pex-tool/actions/expose-pythons@c53dadd8b410bbd66480de91067e9e45d2b3af38
- name: Restore Cached Pyenv Interpreters
if: matrix.cache
id: restore-pyenv-interpreters
uses: actions/cache/restore@v4
with:
path: ${{ env._PEX_TEST_DEV_ROOT }}/pyenv
key: macos-13-${{ runner.arch }}-pex-test-dev-root-pyenv-v1
- name: Restore Cached Devpi Server
if: matrix.cache
id: restore-devpi-server
uses: actions/cache/restore@v4
with:
path: ${{ env._PEX_TEST_DEV_ROOT }}/devpi
# We're using a key suffix / restore-keys prefix trick here to get an updatable cache.
# See: https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
key: macos-13-${{ runner.arch }}-${{ matrix.tox-env }}-pex-test-dev-root-devpi-v1-${{ github.run_id }}
restore-keys: macos-13-${{ runner.arch }}-${{ matrix.tox-env }}-pex-test-dev-root-devpi-v1
# Some ITs need this for VCS URLs of the form git+ssh://[email protected]/...
- name: Setup SSH Agent
uses: webfactory/[email protected]
env:
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
if: env.SSH_PRIVATE_KEY != ''
with:
ssh-private-key: ${{ env.SSH_PRIVATE_KEY }}
- name: Run Tests
uses: pex-tool/actions/run-tox@c53dadd8b410bbd66480de91067e9e45d2b3af38
with:
path: repo/tox.ini
python: ${{ matrix.tox-env-python }}
tox-env: >-
${{ matrix.tox-env }} -- ${{ env._PEX_TEST_POS_ARGS }} ${{ matrix.pex-test-pos-args }}
- name: Cache Pyenv Interpreters
uses: actions/cache/save@v4
if: matrix.cache && github.ref == 'refs/heads/main'
with:
path: ${{ env._PEX_TEST_DEV_ROOT }}/pyenv
key: ${{ steps.restore-pyenv-interpreters.outputs.cache-primary-key }}
- name: Cache Devpi Server
uses: actions/cache/save@v4
if: matrix.cache && github.ref == 'refs/heads/main'
with:
path: ${{ env._PEX_TEST_DEV_ROOT }}/devpi
key: ${{ steps.restore-devpi-server.outputs.cache-primary-key }}
windows-tests:
name: "Windows: tox -e ${{ matrix.tox-env }} ${{ matrix.pex-test-pos-args }}"
needs: setup
runs-on: windows-2022
continue-on-error: true
strategy:
matrix:
include:
# N.B.: We use 3.11 to work around issues building typed-ast for the typecheck step.
# There is a Windows wheel for CPython 3.11, but no higher CPython version.
- python-version: [ 3, 11 ]
tox-env: typecheck,package,docs
- python-version: [ 3, 13 ]
tox-env: py313-pip25_0
artifact-name: unit
cache: true
pex-test-pos-args: ${{ needs.setup.outputs.pex_test_posargs }} --junit-xml ../dist/test-results/unit.xml
- python-version: [ 3, 13 ]
tox-env: py313-pip25_0-integration
artifact-name: integration-1
cache: true
pex-test-pos-args: --shard 1/4 ${{ needs.setup.outputs.pex_test_posargs }} --junit-xml ../dist/test-results/integration-1.xml
- python-version: [ 3, 13 ]
tox-env: py313-pip25_0-integration
artifact-name: integration-2
cache: true
pex-test-pos-args: --shard 2/4 ${{ needs.setup.outputs.pex_test_posargs }} --junit-xml ../dist/test-results/integration-2.xml
- python-version: [ 3, 13 ]
tox-env: py313-pip25_0-integration
artifact-name: integration-3
cache: true
pex-test-pos-args: --shard 3/4 ${{ needs.setup.outputs.pex_test_posargs }} --junit-xml ../dist/test-results/integration-3.xml
- python-version: [ 3, 13 ]
tox-env: py313-pip25_0-integration
artifact-name: integration-4
cache: true
pex-test-pos-args: --shard 4/4 ${{ needs.setup.outputs.pex_test_posargs }} --junit-xml ../dist/test-results/integration-4.xml
steps:
- name: Checkout Pex
uses: actions/checkout@v4
with:
# We need branches and tags for some ITs.
fetch-depth: 0
path: repo
- name: Setup Python ${{ join(matrix.python-version, '.') }}
uses: actions/setup-python@v5
with:
python-version: "${{ join(matrix.python-version, '.') }}"
- name: Restore Cached Devpi Server
if: matrix.cache
id: restore-devpi-server
uses: actions/cache/restore@v4
with:
path: ${{ env._PEX_TEST_DEV_ROOT }}/devpi
# We're using a key suffix / restore-keys prefix trick here to get an updatable cache.
# See: https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
key: windows-2022-${{ runner.arch }}-${{ matrix.tox-env }}-pex-test-dev-root-devpi-v1-${{ github.run_id }}
restore-keys: windows-2022-${{ runner.arch }}-${{ matrix.tox-env }}-pex-test-dev-root-devpi-v1
# Some ITs need this for VCS URLs of the form git+ssh://[email protected]/...
- name: Setup SSH Agent
uses: webfactory/[email protected]
env:
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
if: env.SSH_PRIVATE_KEY != ''
with:
ssh-private-key: ${{ env.SSH_PRIVATE_KEY }}
- name: Run Tests
uses: pex-tool/actions/run-tox@c53dadd8b410bbd66480de91067e9e45d2b3af38
continue-on-error: true
with:
path: repo/tox.ini
tox-env: >-
${{ matrix.tox-env }} -- ${{ matrix.pex-test-pos-args }}
- uses: actions/upload-artifact@v4
if: always() && matrix.artifact-name
with:
name: test-results-${{ matrix.artifact-name }}
path: |
dist/test-results/
.gitignore
- name: Cache Devpi Server
uses: actions/cache/save@v4
if: matrix.cache && github.ref == 'refs/heads/main'
with:
path: ${{ env._PEX_TEST_DEV_ROOT }}/devpi
key: ${{ steps.restore-devpi-server.outputs.cache-primary-key }}
windows-reports:
name: Consolidate Windows Test Results
needs: windows-tests
if: always()
runs-on: ubuntu-24.04
steps:
- uses: actions/download-artifact@v4
with:
merge-multiple: true
- uses: actions/upload-artifact@v4
with:
name: test-results
path: |
dist/test-results/
.gitignore
final-status:
name: Gather Final Status
needs:
- checks
- linux-tests
- mac-tests
- windows-reports
runs-on: ubuntu-24.04
steps:
- name: Check Non-Success
if: |
contains(needs.*.result, 'cancelled') ||
contains(needs.*.result, 'failure') ||
contains(needs.*.result, 'skipped')
run: "false"
- name: Success
run: "true"