Skip to content

Commit 5656360

Browse files
czentgrczentgr
committed
[native] Test standard runner prestocpp build
1 parent ed176b9 commit 5656360

File tree

7 files changed

+415
-0
lines changed

7 files changed

+415
-0
lines changed

‎.github/workflows/format-check.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: format-check
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
format-check:
11+
runs-on: ubuntu-latest
12+
container:
13+
image: public.ecr.aws/oss-presto/velox-dev:check
14+
steps:
15+
- uses: actions/[email protected]
16+
- name: Fix git permissions
17+
# Usually actions/checkout does this but as we run in a container
18+
# it doesn't work
19+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
20+
- name: Check formatting
21+
run: |
22+
git fetch origin master
23+
cd presto-native-execution
24+
make format-check

‎.github/workflows/header-check.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: header-check
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
header-check:
11+
runs-on: ubuntu-latest
12+
container:
13+
image: public.ecr.aws/oss-presto/velox-dev:check
14+
steps:
15+
- uses: actions/[email protected]
16+
- name: Fix git permissions
17+
# Usually actions/checkout does this but as we run in a container
18+
# it doesn't work
19+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
20+
- name: Check license headers
21+
run: |
22+
git fetch origin master
23+
cd presto-native-execution
24+
make header-check

‎.github/workflows/linux-adapters-build.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: linux-adapters-build
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/scripts/**'
8+
9+
jobs:
10+
linux-build-adapters:
11+
runs-on: ubuntu-22.04
12+
container:
13+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
14+
steps:
15+
- uses: actions/[email protected]
16+
- name: Fix git permissions
17+
# Usually actions/checkout does this but as we run in a container
18+
# it doesn't work
19+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
20+
- name: Build all adapter dependencies
21+
run: |
22+
mkdir -p ${GITHUB_WORKSPACE}/adapter-deps/install
23+
source /opt/rh/gcc-toolset-12/enable
24+
set -xu
25+
cd presto-native-execution
26+
DEPENDENCY_DIR=${GITHUB_WORKSPACE}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./velox/scripts/setup-adapters.sh
27+
DEPENDENCY_DIR=${GITHUB_WORKSPACE}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh

‎.github/workflows/linux-build-and-unit-test.yml‎

Lines changed: 243 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,243 @@
1+
2+
name: linux-build-and-unit-test
3+
4+
on:
5+
workflow_dispatch:
6+
pull_request:
7+
8+
jobs:
9+
prestocpp-linux-build-and-unit-test:
10+
runs-on: ubuntu-22.04
11+
container:
12+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
13+
env:
14+
CCACHE_DIR: "${{ github.workspace }}/ccache"
15+
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
16+
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
17+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
18+
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
19+
steps:
20+
- uses: actions/checkout@v4
21+
- name: Fix git permissions
22+
# Usually actions/checkout does this but as we run in a container
23+
# it doesn't work
24+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
25+
26+
- name: Update velox
27+
run: |
28+
cd presto-native-execution
29+
make velox-submodule
30+
31+
- name: Install prestocpp adapter dependencies
32+
run: |
33+
mkdir -p ${HOME}/adapter-deps/install
34+
source /opt/rh/gcc-toolset-12/enable
35+
set -xu
36+
cd presto-native-execution
37+
DEPENDENCY_DIR=${HOME}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh
38+
39+
- name: Install github CLI for the ccache stash
40+
run: ./presto-native-execution/scripts/setup-centos.sh install_gh
41+
42+
- name: Calculate merge-base date for ccache
43+
run: |
44+
git fetch origin master
45+
git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/master HEAD) | tee merge-base-date
46+
47+
- uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
48+
with:
49+
path: '${{ env.CCACHE_DIR }}'
50+
key: ccache-linux-prestocpp-${{ runner.os }}-${{ hashFiles('merge-base-date') }}
51+
52+
- name: Zero ccache statistics
53+
run: ccache -sz
54+
55+
- name: Build engine
56+
run: |
57+
source /opt/rh/gcc-toolset-12/enable
58+
cd presto-native-execution
59+
cmake \
60+
-B _build/release \
61+
-GNinja \
62+
-DTREAT_WARNINGS_AS_ERRORS=1 \
63+
-DENABLE_ALL_WARNINGS=1 \
64+
-DCMAKE_BUILD_TYPE=Release \
65+
-DPRESTO_ENABLE_PARQUET=ON \
66+
-DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \
67+
-DPRESTO_ENABLE_JWT=ON \
68+
-DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \
69+
-DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \
70+
-DCMAKE_PREFIX_PATH=/usr/local \
71+
-DThrift_ROOT=/usr/local \
72+
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
73+
-DMAX_LINK_JOBS=2
74+
ninja -C _build/release -j 4
75+
76+
- name: Ccache after
77+
run: ccache -s
78+
79+
- uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
80+
with:
81+
path: '${{ env.CCACHE_DIR }}'
82+
key: ccache-linux-prestocpp-${{ runner.os }}-${{ hashFiles('merge-base-date') }}
83+
84+
- name: Run Unit Tests
85+
run: |
86+
# Ensure transitive dependency libboost-iostreams is found.
87+
ldconfig /usr/local/lib
88+
cd presto-native-execution/_build/release
89+
ctest -j 4 -VV --output-on-failure --exclude-regex velox.*
90+
91+
- name: Upload artifacts
92+
uses: actions/upload-artifact@v4
93+
with:
94+
name: presto-native-build
95+
path: |
96+
presto-native-execution/_build/release/presto_cpp/main/presto_server
97+
presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main
98+
99+
linux-presto-e2e-tests:
100+
needs: prestocpp-linux-build-and-unit-test
101+
runs-on: ubuntu-22.04
102+
container:
103+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
104+
env:
105+
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
106+
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
107+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
108+
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
109+
steps:
110+
- uses: actions/checkout@v4
111+
- name: Fix git permissions
112+
# Usually actions/checkout does this but as we run in a container
113+
# it doesn't work
114+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
115+
116+
- name: Download artifacts
117+
uses: actions/download-artifact@v4
118+
with:
119+
name: presto-native-build
120+
path: presto-native-execution/_build/release
121+
122+
- name: Install OpenJDK8
123+
uses: actions/setup-java@v4
124+
with:
125+
distribution: 'temurin'
126+
java-version: '8'
127+
128+
- name: Cache local Maven repository
129+
id: cache-maven
130+
uses: actions/cache@v4
131+
with:
132+
path: ~/.m2/repository
133+
key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
134+
restore-keys: |
135+
${{ runner.os }}-maven-2-
136+
137+
- name: Populate maven cache
138+
if: steps.cache-maven.outputs.cache-hit != 'true'
139+
run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
140+
141+
- name: Maven install
142+
env:
143+
MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
144+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
145+
run: |
146+
for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
147+
148+
- name: Run presto-native e2e tests
149+
run: |
150+
export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
151+
export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoNative*.java'`
152+
# Convert file paths to comma separated class names
153+
export TESTCLASSES=
154+
for test_file in $TESTFILES
155+
do
156+
tmp=${test_file##*/}
157+
test_class=${tmp%%\.*}
158+
export TESTCLASSES="${TESTCLASSES},$test_class"
159+
done
160+
export TESTCLASSES=${TESTCLASSES#,}
161+
echo "TESTCLASSES = $TESTCLASSES"
162+
# TODO: neeed to enable remote function tests with
163+
# "-Ppresto-native-execution-remote-functions" once
164+
# > https://github.com/facebookincubator/velox/discussions/6163
165+
# is fixed.
166+
167+
mvn test \
168+
${MAVEN_TEST} \
169+
-pl 'presto-native-execution' \
170+
-Dtest="${TESTCLASSES}" \
171+
-DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
172+
-DDATA_DIR=${RUNNER_TEMP} \
173+
-Duser.timezone=America/Bahia_Banderas \
174+
-T1C
175+
176+
linux-spark-e2e-tests:
177+
needs: prestocpp-linux-build-and-unit-test
178+
runs-on: ubuntu-22.04
179+
container:
180+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
181+
env:
182+
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
183+
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
184+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
185+
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
186+
steps:
187+
- uses: actions/checkout@v4
188+
- name: Fix git permissions
189+
# Usually actions/checkout does this but as we run in a container
190+
# it doesn't work
191+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
192+
193+
- name: Download artifacts
194+
uses: actions/download-artifact@v4
195+
with:
196+
name: presto-native-build
197+
path: presto-native-execution/_build/release
198+
199+
- name: Check Java
200+
run: java --version
201+
202+
- name: Cache local Maven repository
203+
id: cache-maven
204+
uses: actions/cache@v4
205+
with:
206+
path: ~/.m2/repository
207+
key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
208+
restore-keys: |
209+
${{ runner.os }}-maven-2-
210+
211+
- name: Populate maven cache
212+
if: steps.cache-maven.outputs.cache-hit != 'true'
213+
run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
214+
215+
- name: Maven install
216+
env:
217+
MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
218+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
219+
run: |
220+
for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
221+
222+
- name: Run spark e2e tests
223+
run: |
224+
export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
225+
export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoSpark*.java'`
226+
# Convert file paths to comma separated class names
227+
export TESTCLASSES=
228+
for test_file in $TESTFILES
229+
do
230+
tmp=${test_file##*/}
231+
test_class=${tmp%%\.*}
232+
export TESTCLASSES="${TESTCLASSES},$test_class"
233+
done
234+
export TESTCLASSES=${TESTCLASSES#,}
235+
echo "TESTCLASSES = $TESTCLASSES"
236+
mvn test \
237+
${MAVEN_TEST} \
238+
-pl 'presto-native-execution' \
239+
-Dtest="${TESTCLASSES}" \
240+
-DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
241+
-DDATA_DIR=${RUNNER_TEMP} \
242+
-Duser.timezone=America/Bahia_Banderas \
243+
-T1C

‎.github/workflows/macos-build.yml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: macos-build
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
macos-build:
11+
runs-on: macos-13
12+
steps:
13+
- uses: actions/[email protected]
14+
- name: Fix git permissions
15+
# Usually actions/checkout does this but as we run in a container
16+
# it doesn't work
17+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
18+
- name: Update submodules
19+
run: |
20+
git config --global --add safe.directory $(pwd)
21+
cd presto-native-execution
22+
make submodules
23+
- uses: maxim-lobanov/[email protected]
24+
with:
25+
xcode-version: 15
26+
- name: "Setup MacOS"
27+
run: |
28+
set -xu
29+
mkdir ~/deps ~/deps-src
30+
git clone --depth 1 https://github.com/Homebrew/brew ~/deps
31+
PATH=~/deps/bin:${PATH} DEPENDENCY_DIR=~/deps-src INSTALL_PREFIX=~/deps PROMPT_ALWAYS_RESPOND=n ./presto-native-execution/scripts/setup-macos.sh
32+
# Calculate the prefix path before we delete brew's repos and taps.
33+
rm -rf ~/deps/.git ~/deps/Library/Taps/ # Reduce cache size by 70%.
34+
rm -rf ~/deps-src
35+
- name: "Build presto_cpp on MacOS"
36+
run: |
37+
clang --version
38+
export INSTALL_PREFIX=~/deps
39+
export PATH=~/deps/bin:$(brew --prefix m4)/bin:$(brew --prefix bison)/bin:${PATH}
40+
cd presto-native-execution
41+
cmake -B _build/debug -GNinja -DTREAT_WARNINGS_AS_ERRORS=1 -DENABLE_ALL_WARNINGS=1 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
42+
ninja -C _build/debug

0 commit comments

Comments
 (0)