Skip to content

Commit 6d9da21

Browse files
czentgrczentgr
committed
[native] Test standard runner prestocpp build
1 parent ed176b9 commit 6d9da21

File tree

6 files changed

+386
-0
lines changed

6 files changed

+386
-0
lines changed

‎.github/workflows/format-check.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: format-check
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
format-check:
11+
runs-on: ubuntu-latest
12+
container:
13+
image: public.ecr.aws/oss-presto/velox-dev:check
14+
steps:
15+
- uses: actions/[email protected]
16+
- name: Fix git permissions
17+
# Usually actions/checkout does this but as we run in a container
18+
# it doesn't work
19+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
20+
- name: Check formatting
21+
run: |
22+
git fetch origin master
23+
cd presto-native-execution
24+
make format-check

‎.github/workflows/header-check.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: header-check
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
header-check:
11+
runs-on: ubuntu-latest
12+
container:
13+
image: public.ecr.aws/oss-presto/velox-dev:check
14+
steps:
15+
- uses: actions/[email protected]
16+
- name: Fix git permissions
17+
# Usually actions/checkout does this but as we run in a container
18+
# it doesn't work
19+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
20+
- name: Check license headers
21+
run: |
22+
git fetch origin master
23+
cd presto-native-execution
24+
make header-check

‎.github/workflows/linux-build-and-unit-test.yml‎

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
2+
name: linux-build-and-unit-test
3+
4+
on:
5+
workflow_dispatch:
6+
pull_request:
7+
8+
jobs:
9+
linux-build-and-unit-test:
10+
runs-on: ubuntu-22.04
11+
container:
12+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
13+
env:
14+
CCACHE_DIR: "${{ github.workspace }}/.ccache"
15+
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
16+
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
17+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
18+
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
19+
steps:
20+
- uses: actions/checkout@v4
21+
- name: Fix git permissions
22+
# Usually actions/checkout does this but as we run in a container
23+
# it doesn't work
24+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
25+
26+
- name: Update velox
27+
run: |
28+
cd presto-native-execution
29+
make velox-submodule
30+
31+
- name: "Install all adapter dependencies"
32+
run: |
33+
mkdir -p ${HOME}/adapter-deps/install
34+
source /opt/rh/gcc-toolset-12/enable
35+
set -xu
36+
cd presto-native-execution
37+
DEPENDENCY_DIR=${HOME}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh
38+
39+
- name: "Install github CLI for the ccache stash"
40+
run: ./presto-native-execution/scripts/setup-centos.sh install_gh
41+
42+
- name: Calculate merge-base date for CCache
43+
run: git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/master HEAD) | tee merge-base-date
44+
45+
- uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
46+
with:
47+
path: '${{ env.CCACHE_DIR }}'
48+
key: ccache-linux-prestocpp-${{ runner.os }}-${{ hashFiles('merge-base-date') }}
49+
50+
- name: "Zero Ccache Statistics"
51+
run: |
52+
ccache -sz
53+
54+
- name: Build
55+
run: |
56+
source /opt/rh/gcc-toolset-12/enable
57+
cd presto-native-execution
58+
cmake \
59+
-B _build/debug \
60+
-GNinja \
61+
-DTREAT_WARNINGS_AS_ERRORS=1 \
62+
-DENABLE_ALL_WARNINGS=1 \
63+
-DCMAKE_BUILD_TYPE=Debug \
64+
-DPRESTO_ENABLE_PARQUET=ON \
65+
-DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \
66+
-DPRESTO_ENABLE_JWT=ON \
67+
-DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \
68+
-DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \
69+
-DCMAKE_PREFIX_PATH=/usr/local \
70+
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
71+
-DMAX_LINK_JOBS=2
72+
ninja -C _build/debug -j 4
73+
74+
- name: Ccache after
75+
run: ccache -s
76+
77+
- uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
78+
with:
79+
path: '${{ env.CCACHE_DIR }}'
80+
key: ccache-linux-prestocpp-${{ runner.os }}-${{ hashFiles('merge-base-date') }}
81+
82+
- name: Run Unit Tests
83+
run: |
84+
cd presto-native-execution/_build/debug
85+
ctest -j 4 -VV --output-on-failure --exclude-regex velox.*
86+
87+
- name: Upload artifacts
88+
uses: actions/upload-artifact@v4
89+
with:
90+
name: presto-native-build
91+
path: |
92+
presto-native-execution/_build/debug/presto_cpp/main/presto_server
93+
presto-native-execution/_build/debug/velox/velox/functions/remote/server/velox_functions_remote_server_main
94+
95+
linux-presto-e2e-tests:
96+
needs: linux-build-and-unit-test
97+
runs-on: ubuntu-22.04
98+
container:
99+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
100+
env:
101+
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
102+
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
103+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
104+
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
105+
steps:
106+
- uses: actions/checkout@v4
107+
- name: Fix git permissions
108+
# Usually actions/checkout does this but as we run in a container
109+
# it doesn't work
110+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
111+
112+
- name: Download artifacts
113+
uses: actions/download-artifact@v4
114+
with:
115+
name: presto-native-build
116+
path: presto-native-execution/_build/debug
117+
118+
- name: Install OpenJDK8
119+
uses: actions/setup-java@v4
120+
with:
121+
distribution: 'temurin'
122+
java-version: '8'
123+
124+
- name: Cache local Maven repository
125+
id: cache-maven
126+
uses: actions/cache@v4
127+
with:
128+
path: ~/.m2/repository
129+
key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
130+
restore-keys: |
131+
${{ runner.os }}-maven-2-
132+
133+
- name: Populate maven cache
134+
if: steps.cache-maven.outputs.cache-hit != 'true'
135+
run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
136+
137+
- name: Maven install
138+
env:
139+
MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
140+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
141+
run: |
142+
for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
143+
144+
- name: Run presto-native e2e tests
145+
run: |
146+
export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/debug/presto_cpp/main/presto_server"
147+
export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoNative*.java'`
148+
# Convert file paths to comma separated class names
149+
export TESTCLASSES=
150+
for test_file in $TESTFILES
151+
do
152+
tmp=${test_file##*/}
153+
test_class=${tmp%%\.*}
154+
export TESTCLASSES="${TESTCLASSES},$test_class"
155+
done
156+
export TESTCLASSES=${TESTCLASSES#,}
157+
echo "TESTCLASSES = $TESTCLASSES"
158+
# TODO: neeed to enable remote function tests with
159+
# "-Ppresto-native-execution-remote-functions" once
160+
# > https://github.com/facebookincubator/velox/discussions/6163
161+
# is fixed.
162+
163+
mvn test \
164+
${MAVEN_TEST} \
165+
-pl 'presto-native-execution' \
166+
-Dtest="${TESTCLASSES}" \
167+
-DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
168+
-DDATA_DIR=${RUNNER_TEMP} \
169+
-Duser.timezone=America/Bahia_Banderas \
170+
-T1C
171+
172+
linux-spark-e2e-tests:
173+
needs: linux-build-and-unit-test
174+
runs-on: ubuntu-22.04
175+
container:
176+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
177+
env:
178+
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
179+
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
180+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
181+
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
182+
steps:
183+
- uses: actions/checkout@v4
184+
- name: Fix git permissions
185+
# Usually actions/checkout does this but as we run in a container
186+
# it doesn't work
187+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
188+
189+
- name: Download artifacts
190+
uses: actions/download-artifact@v4
191+
with:
192+
name: presto-native-build
193+
path: presto-native-execution/_build/debug
194+
195+
- name: Check Java
196+
run: java --version
197+
198+
- name: Cache local Maven repository
199+
id: cache-maven
200+
uses: actions/cache@v4
201+
with:
202+
path: ~/.m2/repository
203+
key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
204+
restore-keys: |
205+
${{ runner.os }}-maven-2-
206+
207+
- name: Populate maven cache
208+
if: steps.cache-maven.outputs.cache-hit != 'true'
209+
run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
210+
211+
- name: Maven install
212+
env:
213+
MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
214+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
215+
run: |
216+
for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
217+
218+
- name: Run spark e2e tests
219+
run: |
220+
export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/debug/presto_cpp/main/presto_server"
221+
export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoSpark*.java'`
222+
# Convert file paths to comma separated class names
223+
export TESTCLASSES=
224+
for test_file in $TESTFILES
225+
do
226+
tmp=${test_file##*/}
227+
test_class=${tmp%%\.*}
228+
export TESTCLASSES="${TESTCLASSES},$test_class"
229+
done
230+
export TESTCLASSES=${TESTCLASSES#,}
231+
echo "TESTCLASSES = $TESTCLASSES"
232+
mvn test \
233+
${MAVEN_TEST} \
234+
-pl 'presto-native-execution' \
235+
-Dtest="${TESTCLASSES}" \
236+
-DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
237+
-DDATA_DIR=${RUNNER_TEMP} \
238+
-Duser.timezone=America/Bahia_Banderas \
239+
-T1C

‎.github/workflows/macos-build.yml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: macos-build
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
macos-build:
11+
runs-on: macos-13
12+
steps:
13+
- uses: actions/[email protected]
14+
- name: Fix git permissions
15+
# Usually actions/checkout does this but as we run in a container
16+
# it doesn't work
17+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
18+
- name: Update submodules
19+
run: |
20+
git config --global --add safe.directory $(pwd)
21+
cd presto-native-execution
22+
make submodules
23+
- uses: maxim-lobanov/[email protected]
24+
with:
25+
xcode-version: 15
26+
- name: "Setup MacOS"
27+
run: |
28+
set -xu
29+
mkdir ~/deps ~/deps-src
30+
git clone --depth 1 https://github.com/Homebrew/brew ~/deps
31+
PATH=~/deps/bin:${PATH} DEPENDENCY_DIR=~/deps-src INSTALL_PREFIX=~/deps PROMPT_ALWAYS_RESPOND=n ./presto-native-execution/scripts/setup-macos.sh
32+
# Calculate the prefix path before we delete brew's repos and taps.
33+
rm -rf ~/deps/.git ~/deps/Library/Taps/ # Reduce cache size by 70%.
34+
rm -rf ~/deps-src
35+
- name: "Build presto_cpp on MacOS"
36+
run: |
37+
clang --version
38+
export INSTALL_PREFIX=~/deps
39+
export PATH=~/deps/bin:$(brew --prefix m4)/bin:$(brew --prefix bison)/bin:${PATH}
40+
cd presto-native-execution
41+
cmake -B _build/debug -GNinja -DTREAT_WARNINGS_AS_ERRORS=1 -DENABLE_ALL_WARNINGS=1 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
42+
ninja -C _build/debug

‎.github/workflows/prestocpp-linux-build.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: prestocpp-linux-build
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
linux-build-all:
11+
runs-on: ubuntu-22.04
12+
container:
13+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
14+
steps:
15+
- uses: actions/[email protected]
16+
- name: Fix git permissions
17+
# Usually actions/checkout does this but as we run in a container
18+
# it doesn't work
19+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
20+
- name: Update velox
21+
run: |
22+
cd presto-native-execution
23+
make velox-submodule
24+
- name: Install all adapter dependencies
25+
run: |
26+
mkdir -p ${GITHUB_WORKSPACE}/adapter-deps/install
27+
source /opt/rh/gcc-toolset-12/enable
28+
set -xu
29+
cd presto-native-execution
30+
DEPENDENCY_DIR=${GITHUB_WORKSPACE}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./velox/scripts/setup-adapters.sh
31+
DEPENDENCY_DIR=${GITHUB_WORKSPACE}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh
32+
- name: Build All
33+
run: |
34+
source /opt/rh/gcc-toolset-12/enable
35+
cd presto-native-execution
36+
cmake \
37+
-B _build/release \
38+
-GNinja \
39+
-DAWSSDK_ROOT_DIR=${GITHUB_WORKSPACE}/adapter-deps/install \
40+
-DTREAT_WARNINGS_AS_ERRORS=1 \
41+
-DENABLE_ALL_WARNINGS=1 \
42+
-DCMAKE_BUILD_TYPE=Release \
43+
-DPRESTO_ENABLE_PARQUET=ON \
44+
-DPRESTO_ENABLE_S3=ON \
45+
-DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \
46+
-DPRESTO_ENABLE_JWT=ON \
47+
-DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \
48+
-DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \
49+
-DCMAKE_PREFIX_PATH=/usr/local \
50+
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
51+
-DMAX_LINK_JOBS=2
52+
ninja -C _build/release -j 4

‎presto-native-execution/scripts/setup-centos.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,11 @@ function install_proxygen {
5454
cmake_install -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=ON
5555
}
5656

57+
function install_gh {
58+
curl -L https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.rpm > gh_2.63.2_linux_amd64.rpm
59+
rpm -iv gh_2.63.2_linux_amd64.rpm
60+
}
61+
5762
function install_presto_deps {
5863
run_and_time install_presto_deps_from_package_managers
5964
run_and_time install_gperf

0 commit comments

Comments
 (0)