Skip to content

Commit f518759

Browse files
Joe-AbrahamJoe-Abraham
committed
Add e2e test for remote functions
1 parent 4010046 commit f518759

File tree

11 files changed

+478
-19
lines changed

11 files changed

+478
-19
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,3 +66,4 @@ presto-native-execution/deps-install
6666
# Compiled executables used for docker build
6767
/docker/presto-cli-*-executable.jar
6868
/docker/presto-server-*.tar.gz
69+
/docker/presto-remote-function-server-executable.jar

‎docker/Dockerfile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,15 @@ FROM quay.io/centos/centos:stream9
33
ARG PRESTO_VERSION
44
ARG PRESTO_PKG=presto-server-$PRESTO_VERSION.tar.gz
55
ARG PRESTO_CLI_JAR=presto-cli-$PRESTO_VERSION-executable.jar
6+
ARG PRESTO_REMOTE_SERVER_JAR=presto-remote-function-server-executable.jar
67
ARG JMX_PROMETHEUS_JAVAAGENT_VERSION=0.20.0
78

89
ENV PRESTO_HOME="/opt/presto-server"
910

1011
COPY $PRESTO_PKG .
1112
COPY $PRESTO_CLI_JAR /opt/presto-cli
13+
COPY $PRESTO_REMOTE_SERVER_JAR /opt/presto-remote-server
14+
1215

1316
RUN dnf install -y java-11-openjdk less procps python3 \
1417
&& ln -s $(which python3) /usr/bin/python \
@@ -19,6 +22,7 @@ RUN dnf install -y java-11-openjdk less procps python3 \
1922
&& rm -rf ./presto-server-$PRESTO_VERSION \
2023
&& chmod +x /opt/presto-cli \
2124
&& ln -s /opt/presto-cli /usr/local/bin/ \
25+
&& chmod +x /opt/presto-remote-server \
2226
# clean cache jobs
2327
&& mv /etc/yum/protected.d/systemd.conf /etc/yum/protected.d/systemd.conf.bak \
2428
&& dnf clean all \

‎docker/entrypoint.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,6 @@
22

33
set -e
44

5+
java -Dconfig=/opt/function-server/etc/config.properties -jar /opt/presto-remote-server >> log1.txt 2>&1
6+
57
$PRESTO_HOME/bin/launcher run

‎presto-main/pom.xml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -563,6 +563,26 @@
563563
</ignorePackages>
564564
</configuration>
565565
</plugin>
566+
<plugin>
567+
<groupId>org.apache.maven.plugins</groupId>
568+
<artifactId>maven-shade-plugin</artifactId>
569+
<executions>
570+
<execution>
571+
<phase>package</phase>
572+
<goals>
573+
<goal>shade</goal>
574+
</goals>
575+
<configuration>
576+
<finalName>presto-remote-function-server-executable</finalName>
577+
<transformers>
578+
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
579+
<mainClass>com.facebook.presto.server.FunctionServer</mainClass>
580+
</transformer>
581+
</transformers>
582+
</configuration>
583+
</execution>
584+
</executions>
585+
</plugin>
566586
<plugin>
567587
<groupId>org.apache.maven.plugins</groupId>
568588
<artifactId>maven-dependency-plugin</artifactId>

‎presto-native-execution/pom.xml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,12 @@
362362
<include>presto-cli-*-executable.jar</include>
363363
</includes>
364364
</resource>
365+
<resource>
366+
<directory>${project.parent.basedir}/presto-main/target</directory>
367+
<includes>
368+
<include>presto-remote-function-server-executable.jar</include>
369+
</includes>
370+
</resource>
365371
<resource>
366372
<directory>${project.parent.basedir}/presto-server/target</directory>
367373
<includes>
@@ -433,7 +439,7 @@
433439
<args>
434440
<BUILD_TYPE>Release</BUILD_TYPE>
435441
<DEPENDENCY_IMAGE>presto-native-dependency:latest</DEPENDENCY_IMAGE>
436-
<EXTRA_CMAKE_FLAGS>-DPRESTO_ENABLE_TESTING=OFF</EXTRA_CMAKE_FLAGS>
442+
<EXTRA_CMAKE_FLAGS>-DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON</EXTRA_CMAKE_FLAGS>
437443
<NUM_THREADS>2</NUM_THREADS>
438444
<BASE_IMAGE>ubuntu:22.04</BASE_IMAGE>
439445
<OSNAME>ubuntu</OSNAME>

‎presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -315,5 +315,5 @@ JavaClasses:
315315
- presto-main/src/main/java/com/facebook/presto/connector/system/SystemTableLayoutHandle.java
316316
- presto-main/src/main/java/com/facebook/presto/connector/system/SystemTransactionHandle.java
317317
- presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunctionMetadata.java
318-
- presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/JsonBasedUdfFunctionMetadata.java
318+
- presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/JsonBasedUdfFunctionMetadata.java
319319
- presto-spi/src/main/java/com/facebook/presto/spi/plan/ExchangeEncoding.java

‎presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -364,4 +364,4 @@ JavaClasses:
364364
- presto-main/src/main/java/com/facebook/presto/connector/system/SystemTableLayoutHandle.java
365365
- presto-main/src/main/java/com/facebook/presto/connector/system/SystemTransactionHandle.java
366366
- presto-spi/src/main/java/com/facebook/presto/spi/function/AggregationFunctionMetadata.java
367-
- presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/json/JsonBasedUdfFunctionMetadata.java
367+
- presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/JsonBasedUdfFunctionMetadata.java

‎presto-native-execution/src/test/java/com/facebook/presto/nativeworker/ContainerQueryRunner.java‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737

3838
import java.io.IOException;
3939
import java.sql.Connection;
40-
import java.sql.DriverManager;
40+
import java.sql.ResultSet;
4141
import java.sql.SQLException;
4242
import java.sql.Statement;
4343
import java.time.Duration;
@@ -50,6 +50,7 @@
5050
import java.util.logging.Logger;
5151

5252
import static com.facebook.presto.testing.TestingSession.testSessionBuilder;
53+
import static java.sql.DriverManager.getConnection;
5354

5455
public class ContainerQueryRunner
5556
implements QueryRunner
@@ -72,7 +73,6 @@ public class ContainerQueryRunner
7273
private final String schema;
7374
private final int numberOfWorkers;
7475
private Connection connection;
75-
private Statement statement;
7676

7777
public ContainerQueryRunner()
7878
throws InterruptedException, IOException
@@ -108,8 +108,7 @@ public ContainerQueryRunner(int coordinatorPort, String catalog, String schema,
108108
"timeZoneId=UTC");
109109

110110
try {
111-
Connection connection = DriverManager.getConnection(url, "test", null);
112-
statement = connection.createStatement();
111+
connection = getConnection(url, "test", null);
113112
}
114113
catch (SQLException e) {
115114
throw new RuntimeException(e);
@@ -297,12 +296,12 @@ public Session getDefaultSession()
297296
public MaterializedResult execute(Session session, String sql)
298297
{
299298
try {
300-
return ContainerQueryRunnerUtils
301-
.toMaterializedResult(
302-
statement.executeQuery(sql));
299+
Statement statement = connection.createStatement();
300+
ResultSet resultSet = statement.executeQuery(sql);
301+
return ContainerQueryRunnerUtils.toMaterializedResult(resultSet);
303302
}
304303
catch (SQLException e) {
305-
throw new RuntimeException(e);
304+
throw new RuntimeException("Error executing query: " + sql, e);
306305
}
307306
}
308307
}

‎presto-native-execution/src/test/java/com/facebook/presto/nativeworker/ContainerQueryRunnerUtils.java‎

Lines changed: 66 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,19 @@ public static void createNativeWorkerConfigProperties(int coordinatorPort, Strin
8383
createPropertiesFile("testcontainers/" + nodeId + "/etc/config.properties", properties);
8484
}
8585

86+
public static void createNativeWorkerConfigPropertiesWithFunctionServer(int coordinatorPort, int functionServerPort, String nodeId)
87+
throws IOException
88+
{
89+
Properties properties = new Properties();
90+
properties.setProperty("presto.version", "testversion");
91+
properties.setProperty("http-server.http.port", "7777");
92+
properties.setProperty("discovery.uri", "http://presto-coordinator:" + coordinatorPort);
93+
properties.setProperty("system-memory-gb", "2");
94+
properties.setProperty("native.sidecar", "false");
95+
properties.setProperty("remote-function-server.rest.url", "http://presto-coordinator:" + functionServerPort);
96+
createPropertiesFile("testcontainers/" + nodeId + "/etc/config.properties", properties);
97+
}
98+
8699
public static void createCoordinatorConfigProperties(int port)
87100
throws IOException
88101
{
@@ -93,6 +106,8 @@ public static void createCoordinatorConfigProperties(int port)
93106
properties.setProperty("http-server.http.port", Integer.toString(port));
94107
properties.setProperty("discovery-server.enabled", "true");
95108
properties.setProperty("discovery.uri", "http://presto-coordinator:" + port);
109+
properties.setProperty("list-built-in-functions-only", "false");
110+
properties.setProperty("native-execution-enabled", "false");
96111

97112
// Get native worker system properties and add them to the coordinator properties
98113
Map<String, String> nativeWorkerProperties = NativeQueryRunnerUtils.getNativeWorkerSystemProperties();
@@ -103,6 +118,52 @@ public static void createCoordinatorConfigProperties(int port)
103118
createPropertiesFile("testcontainers/coordinator/etc/config.properties", properties);
104119
}
105120

121+
public static void createFunctionNamespaceRemoteProperties()
122+
throws IOException
123+
{
124+
Properties properties = new Properties();
125+
properties.setProperty("function-namespace-manager.name", "rest");
126+
properties.setProperty("supported-function-languages", "Java");
127+
properties.setProperty("function-implementation-type", "REST");
128+
129+
String directoryPath = "testcontainers/function-namespace";
130+
File directory = new File(directoryPath);
131+
if (!directory.exists()) {
132+
directory.mkdirs();
133+
}
134+
135+
createPropertiesFile("testcontainers/coordinator/etc/function-namespace/remote.properties", properties);
136+
}
137+
138+
public static void createFunctionNamespaceRemotePropertiesWithFunctionServer(int functionServerPort)
139+
throws IOException
140+
{
141+
Properties properties = new Properties();
142+
properties.setProperty("function-namespace-manager.name", "rest");
143+
properties.setProperty("supported-function-languages", "Java");
144+
properties.setProperty("function-implementation-type", "REST");
145+
properties.setProperty("rest-based-function-manager.rest.url", "http://localhost:" + functionServerPort);
146+
147+
String directoryPath = "testcontainers/function-namespace";
148+
File directory = new File(directoryPath);
149+
if (!directory.exists()) {
150+
directory.mkdirs();
151+
}
152+
153+
createPropertiesFile("testcontainers/coordinator/etc/function-namespace/remote.properties", properties);
154+
}
155+
156+
public static void createFunctionServerConfigProperties(int functionServerPort)
157+
throws IOException
158+
{
159+
Properties properties = new Properties();
160+
properties.setProperty("http-server.http.port", String.valueOf(functionServerPort));
161+
properties.setProperty("regex-library", "RE2J");
162+
properties.setProperty("parse-decimal-literals-as-double", "true");
163+
164+
createPropertiesFile("testcontainers/coordinator/etc/function-server/config.properties", properties);
165+
}
166+
106167
public static void createCoordinatorJvmConfig()
107168
throws IOException
108169

@@ -161,6 +222,11 @@ public static void createCoordinatorEntryPointScript()
161222
{
162223
String scriptContent = "#!/bin/sh\n" +
163224
"set -e\n" +
225+
"java " +
226+
// "-Dplugin.dir=/opt/presto-remote-server/function-server-plugin " +
227+
// "-Dconfig=/opt/presto-remote-server/function-server-etc/config.properties " +
228+
// "-jar /opt/presto-remote-server >> log1.txt 2>&1 & \n" +
229+
"-Dconfig=/opt/function-server/etc/config.properties -jar /opt/presto-remote-server >> log1.txt 2>&1 & \n" +
164230
"$PRESTO_HOME/bin/launcher run\n";
165231
createScriptFile("testcontainers/coordinator/entrypoint.sh", scriptContent);
166232
}
@@ -204,10 +270,6 @@ public static void createPropertiesFile(String filePath, Properties properties)
204270
parentDir.mkdirs();
205271
}
206272

207-
if (file.exists()) {
208-
throw new IOException("File exists: " + filePath);
209-
}
210-
211273
try (OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8)) {
212274
for (String key : properties.stringPropertyNames()) {
213275
writer.write(key + "=" + properties.getProperty(key) + "\n");
@@ -224,10 +286,6 @@ public static void createScriptFile(String filePath, String scriptContent)
224286
parentDir.mkdirs();
225287
}
226288

227-
if (file.exists()) {
228-
throw new IOException("File exists: " + filePath);
229-
}
230-
231289
try (OutputStream output = new FileOutputStream(file);
232290
OutputStreamWriter writer = new OutputStreamWriter(output, StandardCharsets.UTF_8)) {
233291
writer.write(scriptContent);

0 commit comments

Comments
 (0)