Skip to content

Commit 0eca79c

Browse files
mistercrunchmistercrunch
authored
feat: use docker in frontend GHA to parallelize work (#31490)
1 parent 550d893 commit 0eca79c

File tree

7 files changed

+192
-51
lines changed

7 files changed

+192
-51
lines changed

‎.dockerignore

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@
3434
**/*.sqllite
3535
**/*.swp
3636
**/.terser-plugin-cache/
37-
**/.storybook/
3837
**/node_modules/
3938

4039
tests/

‎.github/workflows/superset-docs-verify.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424
- uses: JustinBeckwith/[email protected]
2525
continue-on-error: true # This will make the job advisory (non-blocking, no red X)
2626
with:
27-
paths: "**/*.md, **/*.mdx"
27+
paths: "**/*.md, **/*.mdx, !superset-frontend/CHANGELOG.md"
2828
linksToSkip: >-
2929
^https://github.com/apache/(superset|incubator-superset)/(pull|issue)/\d+,
3030
http://localhost:8088/,

‎.github/workflows/superset-frontend.yml

Lines changed: 142 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Frontend
1+
name: "Frontend Build CI (unit tests, linting & sanity checks)"
22

33
on:
44
push:
@@ -13,68 +13,166 @@ concurrency:
1313
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
1414
cancel-in-progress: true
1515

16+
env:
17+
TAG: apache/superset:GHA-${{ github.run_id }}
18+
1619
jobs:
1720
frontend-build:
1821
runs-on: ubuntu-24.04
1922
steps:
20-
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
23+
- name: Checkout Code
2124
uses: actions/checkout@v4
2225
with:
2326
persist-credentials: false
24-
submodules: recursive
25-
- name: Check npm lock file version
26-
run: ./scripts/ci_check_npm_lock_version.sh ./superset-frontend/package-lock.json
27-
- name: Check for file changes
27+
28+
- name: Check for File Changes
2829
id: check
2930
uses: ./.github/actions/change-detector/
3031
with:
3132
token: ${{ secrets.GITHUB_TOKEN }}
32-
- name: Setup Node.js
33-
if: steps.check.outputs.frontend
34-
uses: actions/setup-node@v4
35-
with:
36-
node-version: "20"
37-
- name: Install dependencies
38-
if: steps.check.outputs.frontend
39-
uses: ./.github/actions/cached-dependencies
40-
with:
41-
run: npm-install
42-
- name: eslint
43-
if: steps.check.outputs.frontend
44-
working-directory: ./superset-frontend
45-
run: |
46-
npm run eslint -- . --quiet
47-
- name: tsc
33+
34+
- name: Build Docker Image
4835
if: steps.check.outputs.frontend
49-
working-directory: ./superset-frontend
36+
shell: bash
37+
env:
38+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
5039
run: |
51-
npm run type
52-
- name: Build plugins packages
53-
if: steps.check.outputs.frontend
54-
working-directory: ./superset-frontend
55-
run: npm run plugins:build
56-
- name: Build plugins Storybook
57-
if: steps.check.outputs.frontend
58-
working-directory: ./superset-frontend
59-
run: npm run plugins:build-storybook
60-
- name: superset-ui/core coverage
40+
docker buildx build \
41+
-t $TAG \
42+
--cache-from=type=registry,ref=apache/superset-cache:3.10-slim-bookworm \
43+
--target superset-node-ci \
44+
.
45+
46+
- name: Save Docker Image as Artifact
6147
if: steps.check.outputs.frontend
62-
working-directory: ./superset-frontend
6348
run: |
64-
npm run core:cover
65-
- name: unit tests
49+
docker save $TAG | gzip > docker-image.tar.gz
50+
51+
- name: Upload Docker Image Artifact
6652
if: steps.check.outputs.frontend
67-
working-directory: ./superset-frontend
53+
uses: actions/upload-artifact@v4
54+
with:
55+
name: docker-image
56+
path: docker-image.tar.gz
57+
58+
sharded-jest-tests:
59+
needs: frontend-build
60+
if: needs.frontend-build.result == 'success'
61+
strategy:
62+
matrix:
63+
shard: [1, 2, 3, 4, 5, 6, 7, 8]
64+
fail-fast: false
65+
runs-on: ubuntu-24.04
66+
steps:
67+
- name: Download Docker Image Artifact
68+
uses: actions/download-artifact@v4
69+
with:
70+
name: docker-image
71+
72+
- name: Load Docker Image
73+
run: docker load < docker-image.tar.gz
74+
75+
- name: npm run test with coverage
6876
run: |
69-
npm run test -- --coverage --silent
70-
# todo: remove this step when fix generator as a project in root jest.config.js
71-
- name: generator-superset unit tests
72-
if: steps.check.outputs.frontend
73-
working-directory: ./superset-frontend/packages/generator-superset
74-
run: npm run test
75-
- name: Upload code coverage
77+
mkdir -p ${{ github.workspace }}/superset-frontend/coverage
78+
docker run \
79+
-v ${{ github.workspace }}/superset-frontend/coverage:/app/superset-frontend/coverage \
80+
--rm $TAG \
81+
bash -c \
82+
"npm run test -- --coverage --shard=${{ matrix.shard }}/8 --coverageReporters=json-summary"
83+
84+
- name: Upload Coverage Artifact
85+
uses: actions/upload-artifact@v4
86+
with:
87+
name: coverage-artifacts-${{ matrix.shard }}
88+
path: superset-frontend/coverage
89+
90+
report-coverage:
91+
needs: [sharded-jest-tests]
92+
if: needs.frontend-build.result == 'success'
93+
runs-on: ubuntu-24.04
94+
steps:
95+
- name: Download Coverage Artifacts
96+
uses: actions/download-artifact@v4
97+
with:
98+
pattern: coverage-artifacts-*
99+
path: coverage/
100+
101+
- name: Show Files
102+
run: find coverage/
103+
104+
- name: Merge Code Coverage
105+
run: npx nyc merge coverage/ merged-output/coverage-summary.json
106+
107+
- name: Upload Code Coverage
76108
uses: codecov/codecov-action@v5
77109
with:
78110
flags: javascript
79111
token: ${{ secrets.CODECOV_TOKEN }}
80112
verbose: true
113+
files: merged-output/coverage-summary.json
114+
slug: apache/superset
115+
116+
core-cover:
117+
needs: frontend-build
118+
if: needs.frontend-build.result == 'success'
119+
runs-on: ubuntu-24.04
120+
steps:
121+
- name: Download Docker Image Artifact
122+
uses: actions/download-artifact@v4
123+
with:
124+
name: docker-image
125+
126+
- name: Load Docker Image
127+
run: docker load < docker-image.tar.gz
128+
129+
- name: superset-ui/core coverage
130+
run: |
131+
docker run --rm $TAG bash -c \
132+
"npm run core:cover"
133+
134+
lint-frontend:
135+
needs: frontend-build
136+
if: needs.frontend-build.result == 'success'
137+
runs-on: ubuntu-24.04
138+
steps:
139+
- name: Download Docker Image Artifact
140+
uses: actions/download-artifact@v4
141+
with:
142+
name: docker-image
143+
144+
- name: Load Docker Image
145+
run: docker load < docker-image.tar.gz
146+
147+
- name: eslint
148+
run: |
149+
docker run --rm $TAG bash -c \
150+
"npm i && npm run eslint -- . --quiet"
151+
152+
- name: tsc
153+
run: |
154+
docker run --rm $TAG bash -c \
155+
"npm run type"
156+
157+
validate-frontend:
158+
needs: frontend-build
159+
if: needs.frontend-build.result == 'success'
160+
runs-on: ubuntu-24.04
161+
steps:
162+
- name: Download Docker Image Artifact
163+
uses: actions/download-artifact@v4
164+
with:
165+
name: docker-image
166+
167+
- name: Load Docker Image
168+
run: docker load < docker-image.tar.gz
169+
170+
- name: Build Plugins Packages
171+
run: |
172+
docker run --rm $TAG bash -c \
173+
"npm run plugins:build"
174+
175+
- name: Build Plugins Storybook
176+
run: |
177+
docker run --rm $TAG bash -c \
178+
"npm run plugins:build-storybook"

‎Dockerfile

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@ ARG PY_VER=3.10-slim-bookworm
2424
ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64}
2525

2626
######################################################################
27-
# superset-node used for building frontend assets
27+
# superset-node-ci used as a base for building frontend assets and CI
2828
######################################################################
29-
FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node
29+
FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node-ci
3030
ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
3131
ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS}
3232
ARG DEV_MODE="false" # Skip frontend build in dev mode
@@ -53,6 +53,10 @@ RUN mkdir -p /app/superset/static/assets \
5353
/app/superset/translations
5454

5555
# Mount package files and install dependencies if not in dev mode
56+
# NOTE: we mount packages and plugins as they are referenced in package.json as workspaces
57+
# ideally we'd COPY only their package.json. Here npm ci will be cached as long
58+
# as the full content of these folders don't change, yielding a decent cache reuse rate.
59+
# Note that's it's not possible selectively COPY of mount using blobs.
5660
RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.json \
5761
--mount=type=bind,source=./superset-frontend/package-lock.json,target=./package-lock.json \
5862
--mount=type=cache,target=/root/.cache \
@@ -66,6 +70,11 @@ RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.j
6670
# Runs the webpack build process
6771
COPY superset-frontend /app/superset-frontend
6872

73+
######################################################################
74+
# superset-node used for compile frontend assets
75+
######################################################################
76+
FROM superset-node-ci AS superset-node
77+
6978
# Build the frontend if not in dev mode
7079
RUN --mount=type=cache,target=/app/superset-frontend/.temp_cache \
7180
--mount=type=cache,target=/root/.npm \

‎superset-frontend/package-lock.json

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎superset/config.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ def _try_json_readsha(filepath: str, length: int) -> str | None:
136136
# generated on install via setup.py. In the event that we're
137137
# actually running Superset, we will have already installed,
138138
# therefore it WILL exist. When unit tests are running, however,
139-
# it WILL NOT exist, so we fall back to reading package.json
139+
# it WILL NOT exist, so we fall back on reading package.json
140140
VERSION_STRING = _try_json_readversion(VERSION_INFO_FILE) or _try_json_readversion(
141141
PACKAGE_JSON_FILE
142142
)

‎superset/migrations/versions/2025-01-07_16-03_eb1c288c71c4_.py

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
"""empty message
18+
19+
Revision ID: eb1c288c71c4
20+
Revises: ('df3d7e2eb9a4', '7b17aa722e30')
21+
Create Date: 2025-01-07 16:03:44.936921
22+
23+
"""
24+
25+
# revision identifiers, used by Alembic.
26+
revision = "eb1c288c71c4"
27+
down_revision = ("df3d7e2eb9a4", "7b17aa722e30")
28+
29+
30+
def upgrade():
31+
pass
32+
33+
34+
def downgrade():
35+
pass

0 commit comments

Comments
 (0)