Skip to content

Commit 3907e21

Browse files
committed
ci: separate workflows
The dune cache inside 'setup-ocaml' is trimmed to 5GB/jobcount, which is currently about 10. But not all jobs here use the dune cache, so separate the workflows. (Also the dune cache will get compressed afterwards). The SDK build does use the dune cache, but eventually it might become smaller, e.g. if we change it to install just the dependencies needed for SDK gen, and not everything. It does have a lot of unrelated jobs though. The workflows need a unique name, otherwise they cancel each-other. The dune cache is ~1.4GB now compressed, which should speed up the setup-ocaml step on next run as more dependencies will be cached. Setup XenAPI environment now only takes 2m51s instead of 5m55s. Signed-off-by: Edwin Török <[email protected]>
1 parent 342e125 commit 3907e21

File tree

2 files changed

+138
-126
lines changed

2 files changed

+138
-126
lines changed

.github/workflows/main.yml

Lines changed: 0 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -12,109 +12,6 @@ concurrency: # On new push, cancel old workflows from the same PR, branch or tag
1212
cancel-in-progress: true
1313

1414
jobs:
15-
python-test:
16-
name: Python tests
17-
runs-on: ubuntu-22.04
18-
strategy:
19-
fail-fast: false
20-
matrix:
21-
python-version: ["2.7", "3.11"]
22-
steps:
23-
- name: Checkout code
24-
uses: actions/checkout@v4
25-
with:
26-
fetch-depth: 0 # To check which files changed: origin/master..HEAD
27-
- uses: LizardByte/setup-python-action@master
28-
with:
29-
python-version: ${{matrix.python-version}}
30-
31-
- uses: actions/cache@v4
32-
name: Setup cache for running pre-commit fast
33-
with:
34-
path: ~/.cache/pre-commit
35-
key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
36-
37-
- run: echo "::add-matcher::.github/workflows/python-warning-matcher.json"
38-
name: "Setup GitHub for reporting Python warnings as annotations in pull request code review"
39-
40-
- uses: pre-commit/[email protected]
41-
name: Run pre-commit checks (no spaces at end of lines, etc)
42-
if: ${{ matrix.python-version != '2.7' }}
43-
with:
44-
extra_args: --all-files --verbose --hook-stage commit
45-
env:
46-
SKIP: no-commit-to-branch
47-
48-
- name: Install dependencies only needed for python 2
49-
if: ${{ matrix.python-version == '2.7' }}
50-
run: pip install enum
51-
52-
- name: Install dependencies only needed for python 3
53-
if: ${{ matrix.python-version != '2.7' }}
54-
run: pip install opentelemetry-api opentelemetry-exporter-zipkin-json opentelemetry-sdk pandas pytype toml wrapt
55-
56-
- name: Install common dependencies for Python ${{matrix.python-version}}
57-
run: pip install future mock pytest-coverage pytest-mock
58-
59-
- name: Run Pytest for python 2 and get code coverage for Codecov
60-
if: ${{ matrix.python-version == '2.7' }}
61-
run: >
62-
pytest
63-
--cov=scripts --cov=ocaml/xcp-rrdd
64-
scripts/ ocaml/xcp-rrdd -vv -rA
65-
--junitxml=.git/pytest${{matrix.python-version}}.xml
66-
--cov-report term-missing
67-
--cov-report xml:.git/coverage${{matrix.python-version}}.xml
68-
env:
69-
PYTHONDEVMODE: yes
70-
71-
- name: Run Pytest for python 3 and get code coverage for Codecov
72-
if: ${{ matrix.python-version != '2.7' }}
73-
run: >
74-
pytest
75-
--cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/
76-
scripts/ ocaml/xcp-rrdd python3/ -vv -rA
77-
--junitxml=.git/pytest${{matrix.python-version}}.xml
78-
--cov-report term-missing
79-
--cov-report xml:.git/coverage${{matrix.python-version}}.xml
80-
env:
81-
PYTHONDEVMODE: yes
82-
83-
- name: Upload Python ${{matrix.python-version}} coverage report to Codecov
84-
uses: codecov/codecov-action@v3
85-
with:
86-
directory: .git
87-
files: coverage${{matrix.python-version}}.xml
88-
env_vars: OS,PYTHON
89-
fail_ci_if_error: false
90-
flags: python${{matrix.python-version}}
91-
name: coverage${{matrix.python-version}}
92-
verbose: true
93-
94-
- uses: dciborow/[email protected]
95-
if: ${{ matrix.python-version != '2.7' }}
96-
with:
97-
reporter: github-pr-review
98-
level: warning
99-
# To be customized to cover remaining Python scripts:
100-
glob_pattern: "**/*.py"
101-
continue-on-error: true
102-
103-
- name: Run pytype checks
104-
if: ${{ matrix.python-version != '2.7' }}
105-
run: ./pytype_reporter.py
106-
env:
107-
PR_NUMBER: ${{ github.event.number }}
108-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
109-
PYTYPE_REPORTER_DEBUG: True
110-
111-
- name: pyflakes
112-
uses: reviewdog/action-pyflakes@v1
113-
with:
114-
github_token: ${{ secrets.github_token }}
115-
continue-on-error: true
116-
117-
11815
ocaml-tests:
11916
name: Run OCaml tests
12017
runs-on: ubuntu-22.04
@@ -152,26 +49,3 @@ jobs:
15249
15350
- name: Check disk space
15451
run: df -h || true
155-
156-
deprecation-test:
157-
name: Deprecation tests
158-
runs-on: ubuntu-22.04
159-
160-
steps:
161-
- name: Checkout code
162-
uses: actions/checkout@v4
163-
164-
- name: Generate empty configuration for make to be happy
165-
run: touch config.mk
166-
167-
- name: quality-gate
168-
run: make quality-gate
169-
170-
test-sdk-builds:
171-
name: Test SDK builds
172-
uses: ./.github/workflows/generate-and-build-sdks.yml
173-
with:
174-
# Ensure you also update ocaml-tests
175-
# when changing this value, to keep builds
176-
# consistent
177-
xapi_version: "v0.0.0"

.github/workflows/other.yml

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
name: Build and test (other)
2+
3+
on:
4+
push:
5+
pull_request:
6+
schedule:
7+
# run daily, this refreshes the cache
8+
- cron: "13 2 * * *"
9+
10+
concurrency: # On new push, cancel old workflows from the same PR, branch or tag:
11+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
12+
cancel-in-progress: true
13+
14+
jobs:
15+
python-test:
16+
name: Python tests
17+
runs-on: ubuntu-22.04
18+
strategy:
19+
fail-fast: false
20+
matrix:
21+
python-version: ["2.7", "3.11"]
22+
steps:
23+
- name: Checkout code
24+
uses: actions/checkout@v4
25+
with:
26+
fetch-depth: 0 # To check which files changed: origin/master..HEAD
27+
- uses: LizardByte/setup-python-action@master
28+
with:
29+
python-version: ${{matrix.python-version}}
30+
31+
- uses: actions/cache@v4
32+
name: Setup cache for running pre-commit fast
33+
with:
34+
path: ~/.cache/pre-commit
35+
key: pre-commit|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
36+
37+
- run: echo "::add-matcher::.github/workflows/python-warning-matcher.json"
38+
name: "Setup GitHub for reporting Python warnings as annotations in pull request code review"
39+
40+
- uses: pre-commit/[email protected]
41+
name: Run pre-commit checks (no spaces at end of lines, etc)
42+
if: ${{ matrix.python-version != '2.7' }}
43+
with:
44+
extra_args: --all-files --verbose --hook-stage commit
45+
env:
46+
SKIP: no-commit-to-branch
47+
48+
- name: Install dependencies only needed for python 2
49+
if: ${{ matrix.python-version == '2.7' }}
50+
run: pip install enum
51+
52+
- name: Install dependencies only needed for python 3
53+
if: ${{ matrix.python-version != '2.7' }}
54+
run: pip install opentelemetry-api opentelemetry-exporter-zipkin-json opentelemetry-sdk pandas pytype toml wrapt
55+
56+
- name: Install common dependencies for Python ${{matrix.python-version}}
57+
run: pip install future mock pytest-coverage pytest-mock
58+
59+
- name: Run Pytest for python 2 and get code coverage for Codecov
60+
if: ${{ matrix.python-version == '2.7' }}
61+
run: >
62+
pytest
63+
--cov=scripts --cov=ocaml/xcp-rrdd
64+
scripts/ ocaml/xcp-rrdd -vv -rA
65+
--junitxml=.git/pytest${{matrix.python-version}}.xml
66+
--cov-report term-missing
67+
--cov-report xml:.git/coverage${{matrix.python-version}}.xml
68+
env:
69+
PYTHONDEVMODE: yes
70+
71+
- name: Run Pytest for python 3 and get code coverage for Codecov
72+
if: ${{ matrix.python-version != '2.7' }}
73+
run: >
74+
pytest
75+
--cov=scripts --cov=ocaml/xcp-rrdd --cov=python3/
76+
scripts/ ocaml/xcp-rrdd python3/ -vv -rA
77+
--junitxml=.git/pytest${{matrix.python-version}}.xml
78+
--cov-report term-missing
79+
--cov-report xml:.git/coverage${{matrix.python-version}}.xml
80+
env:
81+
PYTHONDEVMODE: yes
82+
83+
- name: Upload Python ${{matrix.python-version}} coverage report to Codecov
84+
uses: codecov/codecov-action@v3
85+
with:
86+
directory: .git
87+
files: coverage${{matrix.python-version}}.xml
88+
env_vars: OS,PYTHON
89+
fail_ci_if_error: false
90+
flags: python${{matrix.python-version}}
91+
name: coverage${{matrix.python-version}}
92+
verbose: true
93+
94+
- uses: dciborow/[email protected]
95+
if: ${{ matrix.python-version != '2.7' }}
96+
with:
97+
reporter: github-pr-review
98+
level: warning
99+
# To be customized to cover remaining Python scripts:
100+
glob_pattern: "**/*.py"
101+
continue-on-error: true
102+
103+
- name: Run pytype checks
104+
if: ${{ matrix.python-version != '2.7' }}
105+
run: ./pytype_reporter.py
106+
env:
107+
PR_NUMBER: ${{ github.event.number }}
108+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
109+
PYTYPE_REPORTER_DEBUG: True
110+
111+
- name: pyflakes
112+
uses: reviewdog/action-pyflakes@v1
113+
with:
114+
github_token: ${{ secrets.github_token }}
115+
continue-on-error: true
116+
117+
deprecation-test:
118+
name: Deprecation tests
119+
runs-on: ubuntu-22.04
120+
121+
steps:
122+
- name: Checkout code
123+
uses: actions/checkout@v4
124+
125+
- name: Generate empty configuration for make to be happy
126+
run: touch config.mk
127+
128+
- name: quality-gate
129+
run: make quality-gate
130+
131+
test-sdk-builds:
132+
name: Test SDK builds
133+
uses: ./.github/workflows/generate-and-build-sdks.yml
134+
with:
135+
# Ensure you also update ocaml-tests
136+
# when changing this value, to keep builds
137+
# consistent
138+
xapi_version: "v0.0.0"

0 commit comments

Comments
 (0)