Skip to content

Commit b309402

Browse files
tdcmeehantdcmeehan
committed
Add support for custom connector-provided serialization codecs
1 parent 2b76d0b commit b309402

File tree

34 files changed

+2459
-56
lines changed

34 files changed

+2459
-56
lines changed

‎pom.xml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2852,6 +2852,26 @@
28522852
<version>0.5.1</version>
28532853
</plugin>
28542854

2855+
<plugin>
2856+
<groupId>pl.project13.maven</groupId>
2857+
<artifactId>git-commit-id-plugin</artifactId>
2858+
<version>4.0.3</version>
2859+
<configuration>
2860+
<useNativeGit>true</useNativeGit>
2861+
<!-- Include only properties used above to speed up build (https://github.com/git-commit-id/git-commit-id-maven-plugin/issues/462) -->
2862+
<includeOnlyProperties>
2863+
<includeOnlyProperty>\Qgit.build.time</includeOnlyProperty>
2864+
<includeOnlyProperty>\Qgit.commit.id</includeOnlyProperty>
2865+
<includeOnlyProperty>\Qgit.commit.id.describe</includeOnlyProperty>
2866+
<includeOnlyProperty>\Qgit.commit.id.abbrev</includeOnlyProperty>
2867+
</includeOnlyProperties>
2868+
<dateFormat>yyyy-MM-dd'T'HH:mm:ssZZ</dateFormat>
2869+
<gitDescribe>
2870+
<tags>true</tags>
2871+
</gitDescribe>
2872+
</configuration>
2873+
</plugin>
2874+
28552875
</plugins>
28562876
</pluginManagement>
28572877

‎presto-delta/src/test/java/com/facebook/presto/delta/TestDeltaTableHandle.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@
2525
import com.facebook.presto.common.type.Type;
2626
import com.facebook.presto.common.type.TypeManager;
2727
import com.facebook.presto.metadata.FunctionAndTypeManager;
28-
import com.facebook.presto.metadata.HandleJsonModule;
2928
import com.facebook.presto.metadata.HandleResolver;
29+
import com.facebook.presto.metadata.TestingHandleJsonModule;
3030
import com.facebook.presto.sql.analyzer.FeaturesConfig;
3131
import com.facebook.presto.type.TypeDeserializer;
3232
import com.google.common.collect.ImmutableList;
@@ -91,7 +91,7 @@ private JsonCodec<DeltaTableHandle> getJsonCodec()
9191
{
9292
Module module = binder -> {
9393
binder.install(new JsonModule());
94-
binder.install(new HandleJsonModule());
94+
binder.install(new TestingHandleJsonModule());
9595
configBinder(binder).bindConfig(FeaturesConfig.class);
9696
FunctionAndTypeManager functionAndTypeManager = createTestFunctionAndTypeManager();
9797
binder.bind(TypeManager.class).toInstance(functionAndTypeManager);

‎presto-docs/src/main/sphinx/admin/properties.rst‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -571,6 +571,24 @@ shared across all of the partitioned consumers. Increasing this value may
571571
improve network throughput for data transferred between stages if the
572572
network has high latency or if there are many nodes in the cluster.
573573

574+
``use-connector-provided-serialization-codecs``
575+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
576+
577+
* **Type:** ``boolean``
578+
* **Default value:** ``false``
579+
580+
Enables the use of custom connector-provided serialization codecs for handles.
581+
This feature allows connectors to use their own serialization format for
582+
handle objects (such as table handles, column handles, and splits) instead
583+
of standard JSON serialization.
584+
585+
When enabled, connectors that provide a ``ConnectorCodecProvider`` with
586+
appropriate codecs will have their handles serialized using custom binary
587+
formats, which are then Base64-encoded for transport. Connectors without
588+
codec support automatically fall back to standard JSON serialization.
589+
Internal Presto handles (prefixed with ``$``) always use JSON serialization
590+
regardless of this setting.
591+
574592
.. _task-properties:
575593

576594
Task Properties

‎presto-hive/src/test/java/com/facebook/presto/hive/TestHiveSplit.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@
2727
import com.facebook.presto.hive.metastore.Storage;
2828
import com.facebook.presto.hive.metastore.StorageFormat;
2929
import com.facebook.presto.metadata.FunctionAndTypeManager;
30-
import com.facebook.presto.metadata.HandleJsonModule;
3130
import com.facebook.presto.metadata.HandleResolver;
31+
import com.facebook.presto.metadata.TestingHandleJsonModule;
3232
import com.facebook.presto.spi.ColumnHandle;
3333
import com.facebook.presto.spi.HostAddress;
3434
import com.facebook.presto.spi.SplitWeight;
@@ -153,8 +153,8 @@ private JsonCodec<HiveSplit> getJsonCodec()
153153
{
154154
Module module = binder -> {
155155
binder.install(new JsonModule());
156-
binder.install(new HandleJsonModule());
157156
configBinder(binder).bindConfig(FeaturesConfig.class);
157+
binder.install(new TestingHandleJsonModule());
158158
FunctionAndTypeManager functionAndTypeManager = createTestFunctionAndTypeManager();
159159
binder.bind(TypeManager.class).toInstance(functionAndTypeManager);
160160
jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class);

‎presto-main-base/src/main/java/com/facebook/presto/connector/ConnectorManager.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -393,6 +393,16 @@ private Connector createConnector(ConnectorId connectorId, ConnectorFactory fact
393393
}
394394
}
395395

396+
public Optional<ConnectorCodecProvider> getConnectorCodecProvider(ConnectorId connectorId)
397+
{
398+
requireNonNull(connectorId, "connectorId is null");
399+
MaterializedConnector materializedConnector = connectors.get(connectorId);
400+
if (materializedConnector == null) {
401+
return Optional.empty();
402+
}
403+
return materializedConnector.getConnectorCodecProvider();
404+
}
405+
396406
private static class MaterializedConnector
397407
{
398408
private final ConnectorId connectorId;

‎presto-main-base/src/main/java/com/facebook/presto/index/IndexHandleJacksonModule.java‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,47 @@
1313
*/
1414
package com.facebook.presto.index;
1515

16+
import com.facebook.presto.connector.ConnectorManager;
1617
import com.facebook.presto.metadata.AbstractTypedJacksonModule;
1718
import com.facebook.presto.metadata.HandleResolver;
19+
import com.facebook.presto.spi.ConnectorCodec;
20+
import com.facebook.presto.spi.ConnectorId;
1821
import com.facebook.presto.spi.ConnectorIndexHandle;
22+
import com.facebook.presto.spi.connector.ConnectorCodecProvider;
23+
import com.facebook.presto.sql.analyzer.FeaturesConfig;
1924
import jakarta.inject.Inject;
25+
import jakarta.inject.Provider;
26+
27+
import java.util.Optional;
28+
import java.util.function.Function;
2029

2130
public class IndexHandleJacksonModule
2231
extends AbstractTypedJacksonModule<ConnectorIndexHandle>
2332
{
2433
@Inject
25-
public IndexHandleJacksonModule(HandleResolver handleResolver)
34+
public IndexHandleJacksonModule(
35+
HandleResolver handleResolver,
36+
Provider<ConnectorManager> connectorManagerProvider,
37+
FeaturesConfig featuresConfig)
38+
{
39+
super(ConnectorIndexHandle.class,
40+
handleResolver::getId,
41+
handleResolver::getIndexHandleClass,
42+
featuresConfig.isUseConnectorProvidedSerializationCodecs(),
43+
connectorId -> connectorManagerProvider.get()
44+
.getConnectorCodecProvider(connectorId)
45+
.flatMap(ConnectorCodecProvider::getConnectorIndexHandleCodec));
46+
}
47+
48+
public IndexHandleJacksonModule(
49+
HandleResolver handleResolver,
50+
FeaturesConfig featuresConfig,
51+
Function<ConnectorId, Optional<ConnectorCodec<ConnectorIndexHandle>>> codecExtractor)
2652
{
2753
super(ConnectorIndexHandle.class,
2854
handleResolver::getId,
29-
handleResolver::getIndexHandleClass);
55+
handleResolver::getIndexHandleClass,
56+
featuresConfig.isUseConnectorProvidedSerializationCodecs(),
57+
codecExtractor);
3058
}
3159
}

0 commit comments

Comments
 (0)