Node TTL Fetchers and Cluster TTL Providers

Author: neeradsomanchi

pom.xml

@@ -172,6 +172,8 @@
         <module>presto-thrift-testing-udf-server</module>
         <module>presto-thrift-spec</module>
         <module>presto-testng-services</module>
+        <module>presto-node-ttl-fetchers</module>
+        <module>presto-cluster-ttl-providers</module>
     </modules>
 
     <dependencyManagement>
@@ -233,6 +235,18 @@
                 <version>${project.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>com.facebook.presto</groupId>
+                <artifactId>presto-node-ttl-fetchers</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+
+            <dependency>
+                <groupId>com.facebook.presto</groupId>
+                <artifactId>presto-cluster-ttl-providers</artifactId>
+                <version>${project.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>com.facebook.presto</groupId>
                 <artifactId>presto-array</artifactId>

presto-cluster-ttl-providers/pom.xml

@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>presto-root</artifactId>
+        <groupId>com.facebook.presto</groupId>
+        <version>0.262-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>presto-cluster-ttl-providers</artifactId>
+    <description>Presto - Cluster Ttl Providers</description>
+    <packaging>presto-plugin</packaging>
+
+    <properties>
+        <air.main.basedir>${project.parent.basedir}</air.main.basedir>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.inject</groupId>
+            <artifactId>guice</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.facebook.airlift</groupId>
+            <artifactId>configuration</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.facebook.airlift</groupId>
+            <artifactId>bootstrap</artifactId>
+        </dependency>
+
+        <!-- Presto SPI -->
+        <dependency>
+            <groupId>com.facebook.presto</groupId>
+            <artifactId>presto-spi</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>io.airlift</groupId>
+            <artifactId>units</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.facebook.presto</groupId>
+            <artifactId>presto-common</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.testng</groupId>
+            <artifactId>testng</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+</project>

presto-cluster-ttl-providers/src/main/java/com/facebook/presto/clusterttlproviders/ClusterTtlProviderPlugin.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.clusterttlproviders;
+
+import com.facebook.presto.clusterttlproviders.infinite.InfiniteClusterTtlProviderFactory;
+import com.facebook.presto.clusterttlproviders.percentile.PercentileClusterTtlProviderFactory;
+import com.facebook.presto.spi.Plugin;
+import com.facebook.presto.spi.ttl.ClusterTtlProviderFactory;
+import com.google.common.collect.ImmutableList;
+
+public class ClusterTtlProviderPlugin
+        implements Plugin
+{
+    @Override
+    public Iterable<ClusterTtlProviderFactory> getClusterTtlProviderFactories()
+    {
+        return ImmutableList.of(new InfiniteClusterTtlProviderFactory(), new PercentileClusterTtlProviderFactory());
+    }
+}

presto-cluster-ttl-providers/src/main/java/com/facebook/presto/clusterttlproviders/infinite/InfiniteClusterTtlProvider.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.facebook.presto.clusterttlproviders.infinite;
+
+import com.facebook.presto.spi.ttl.ClusterTtlProvider;
+import com.facebook.presto.spi.ttl.ConfidenceBasedTtlInfo;
+import com.facebook.presto.spi.ttl.NodeTtl;
+
+import java.util.List;
+
+import static com.facebook.presto.spi.ttl.ConfidenceBasedTtlInfo.getInfiniteTtl;
+
+public class InfiniteClusterTtlProvider
+        implements ClusterTtlProvider
+{
+    @Override
+    public ConfidenceBasedTtlInfo getClusterTtl(List<NodeTtl> nodeTtls)
+    {
+        return getInfiniteTtl();
+    }
+}

presto-cluster-ttl-providers/src/main/java/com/facebook/presto/clusterttlproviders/infinite/InfiniteClusterTtlProviderFactory.java

@@ -0,0 +1,35 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.clusterttlproviders.infinite;
+
+import com.facebook.presto.spi.ttl.ClusterTtlProvider;
+import com.facebook.presto.spi.ttl.ClusterTtlProviderFactory;
+
+import java.util.Map;
+
+public class InfiniteClusterTtlProviderFactory
+        implements ClusterTtlProviderFactory
+{
+    @Override
+    public String getName()
+    {
+        return "infinite";
+    }
+
+    @Override
+    public ClusterTtlProvider create(Map<String, String> config)
+    {
+        return new InfiniteClusterTtlProvider();
+    }
+}

presto-cluster-ttl-providers/src/main/java/com/facebook/presto/clusterttlproviders/percentile/PercentileBasedClusterTtlProvider.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.facebook.presto.clusterttlproviders.percentile;
+
+import com.facebook.presto.spi.ttl.ClusterTtlProvider;
+import com.facebook.presto.spi.ttl.ConfidenceBasedTtlInfo;
+import com.facebook.presto.spi.ttl.NodeTtl;
+import com.google.inject.Inject;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.Objects.requireNonNull;
+import static java.util.stream.Collectors.toList;
+
+public class PercentileBasedClusterTtlProvider
+        implements ClusterTtlProvider
+{
+    private final Comparator<ConfidenceBasedTtlInfo> ttlComparator = Comparator.comparing(ConfidenceBasedTtlInfo::getExpiryInstant);
+    private final int percentile;
+
+    @Inject
+    public PercentileBasedClusterTtlProvider(PercentileBasedClusterTtlProviderConfig config)
+    {
+        requireNonNull(config, "config is null");
+        checkArgument(config.getPercentile() > 0, "percentile should be greater than 0");
+        this.percentile = config.getPercentile();
+    }
+
+    @Override
+    public ConfidenceBasedTtlInfo getClusterTtl(List<NodeTtl> nodeTtls)
+    {
+        List<ConfidenceBasedTtlInfo> ttls = nodeTtls.stream()
+                .map(this::getMinTtl)
+                .filter(Optional::isPresent)
+                .map(Optional::get)
+                .sorted(ttlComparator)
+                .collect(toList());
+
+        if (ttls.size() == 0) {
+            return new ConfidenceBasedTtlInfo(0, 100);
+        }
+
+        return ttls.get((int) Math.ceil(ttls.size() * percentile / 100.0) - 1);
+    }
+
+    private Optional<ConfidenceBasedTtlInfo> getMinTtl(NodeTtl nodeTtl)
+    {
+        return nodeTtl.getTtlInfo().stream().min(ttlComparator);
+    }
+}

presto-cluster-ttl-providers/src/main/java/com/facebook/presto/clusterttlproviders/percentile/PercentileBasedClusterTtlProviderConfig.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.clusterttlproviders.percentile;
+
+import com.facebook.airlift.configuration.Config;
+
+public class PercentileBasedClusterTtlProviderConfig
+{
+    private int percentile = 1;
+
+    public int getPercentile()
+    {
+        return percentile;
+    }
+
+    @Config("percentile")
+    public PercentileBasedClusterTtlProviderConfig setPercentile(int percentile)
+    {
+        this.percentile = percentile;
+        return this;
+    }
+}

presto-cluster-ttl-providers/src/main/java/com/facebook/presto/clusterttlproviders/percentile/PercentileBasedClusterTtlProviderModule.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.clusterttlproviders.percentile;
+
+import com.facebook.presto.spi.ttl.ClusterTtlProvider;
+import com.google.inject.Binder;
+import com.google.inject.Module;
+import com.google.inject.Scopes;
+
+import static com.facebook.airlift.configuration.ConfigBinder.configBinder;
+
+public class PercentileBasedClusterTtlProviderModule
+        implements Module
+{
+    @Override
+    public void configure(Binder binder)
+    {
+        configBinder(binder).bindConfig(PercentileBasedClusterTtlProviderConfig.class, "cluster-ttl-provider");
+        binder.bind(ClusterTtlProvider.class).to(PercentileBasedClusterTtlProvider.class).in(Scopes.SINGLETON);
+    }
+}

presto-cluster-ttl-providers/src/main/java/com/facebook/presto/clusterttlproviders/percentile/PercentileClusterTtlProviderFactory.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.clusterttlproviders.percentile;
+
+import com.facebook.airlift.bootstrap.Bootstrap;
+import com.facebook.presto.spi.ttl.ClusterTtlProvider;
+import com.facebook.presto.spi.ttl.ClusterTtlProviderFactory;
+import com.google.inject.Injector;
+
+import java.util.Map;
+
+public class PercentileClusterTtlProviderFactory
+        implements ClusterTtlProviderFactory
+{
+    @Override
+    public String getName()
+    {
+        return "percentile";
+    }
+
+    @Override
+    public ClusterTtlProvider create(Map<String, String> config)
+    {
+        try {
+            Bootstrap app = new Bootstrap(new PercentileBasedClusterTtlProviderModule());
+            Injector injector = app
+                    .doNotInitializeLogging()
+                    .setRequiredConfigurationProperties(config)
+                    .initialize();
+            return injector.getInstance(ClusterTtlProvider.class);
+        }
+        catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}