Use Hive 4 and Hadoop 3 images

Author: imjalpreet

presto-hive-hadoop2/bin/common.sh

@@ -67,8 +67,8 @@ function termination_handler(){
   exit 130
 }
 
-export HADOOP_BASE_IMAGE="${HADOOP_BASE_IMAGE:-prestodb/hdp2.6-hive}"
-export DOCKER_IMAGES_VERSION=${DOCKER_IMAGES_VERSION:-5}
+export HADOOP_BASE_IMAGE="${HADOOP_BASE_IMAGE:-imjalpreet/hdp3.1-hive}"
+export DOCKER_IMAGES_VERSION=${DOCKER_IMAGES_VERSION:-latest}
 
 SCRIPT_DIR="${BASH_SOURCE%/*}"
 INTEGRATION_TESTS_ROOT="${SCRIPT_DIR}/.."

presto-hive/pom.xml

@@ -554,6 +554,7 @@
                                 <exclude>**/TestParquetDistributedQueries.java</exclude>
                                 <exclude>**/TestHive2InsertOverwrite.java</exclude>
                                 <exclude>**/TestHive3InsertOverwrite.java</exclude>
+                                <exclude>**/TestHive4InsertOverwrite.java</exclude>
                             </excludes>
                         </configuration>
                     </plugin>
@@ -700,6 +701,7 @@
                             <includes>
                                 <include>**/TestHive2InsertOverwrite.java</include>
                                 <include>**/TestHive3InsertOverwrite.java</include>
+                                <include>**/TestHive4InsertOverwrite.java</include>
                             </includes>
                         </configuration>
                     </plugin>

presto-hive/src/test/java/com/facebook/presto/hive/AbstractHiveSslTest.java

@@ -25,7 +25,7 @@
 import java.util.Map;
 
 import static com.facebook.airlift.testing.Closeables.closeAllRuntimeException;
-import static com.facebook.presto.hive.containers.HiveHadoopContainer.HIVE3_IMAGE;
+import static com.facebook.presto.hive.containers.HiveHadoopContainer.HIVE4_IMAGE;
 import static com.facebook.presto.tests.sql.TestTable.randomTableSuffix;
 import static java.lang.String.format;
 
@@ -46,7 +46,7 @@ public abstract class AbstractHiveSslTest
     protected QueryRunner createQueryRunner() throws Exception
     {
         this.bucketName = "test-hive-ssl-enable-" + randomTableSuffix();
-        this.dockerizedS3DataLake = new HiveMinIODataLake(bucketName, ImmutableMap.of(), HIVE3_IMAGE, true);
+        this.dockerizedS3DataLake = new HiveMinIODataLake(bucketName, ImmutableMap.of(), HIVE4_IMAGE, true);
         this.dockerizedS3DataLake.start();
         return S3HiveQueryRunner.create(
                 this.dockerizedS3DataLake.getHiveHadoop().getHiveMetastoreEndpoint(),

presto-hive/src/test/java/com/facebook/presto/hive/TestHive4InsertOverwrite.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.hive;
+
+import static com.facebook.presto.hive.containers.HiveHadoopContainer.HIVE4_IMAGE;
+
+public class TestHive4InsertOverwrite
+        extends BaseTestHiveInsertOverwrite
+{
+    public TestHive4InsertOverwrite()
+    {
+        super(HIVE4_IMAGE);
+    }
+}

presto-hive/src/test/java/com/facebook/presto/hive/containers/HiveHadoopContainer.java

@@ -30,9 +30,10 @@ public class HiveHadoopContainer
 {
     private static final Logger log = Logger.get(HiveHadoopContainer.class);
 
-    private static final String IMAGE_VERSION = "10";
-    public static final String DEFAULT_IMAGE = "prestodb/hdp2.6-hive:" + IMAGE_VERSION;
-    public static final String HIVE3_IMAGE = "prestodb/hive3.1-hive:" + IMAGE_VERSION;
+    private static final String IMAGE_VERSION = "latest";
+    public static final String DEFAULT_IMAGE = "imjalpreet/hdp3.1-hive:" + IMAGE_VERSION;
+    public static final String HIVE3_IMAGE = "prestodb/hive3.1-hive:10";
+    public static final String HIVE4_IMAGE = "imjalpreet/hive4.0-hive:" + IMAGE_VERSION;
 
     public static final String HOST_NAME = "hadoop-master";
 

presto-hive/src/test/java/com/facebook/presto/hive/containers/HiveMinIODataLake.java

@@ -26,9 +26,11 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.Map;
+import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static com.facebook.presto.hive.containers.HiveHadoopContainer.HIVE3_IMAGE;
+import static com.facebook.presto.hive.containers.HiveHadoopContainer.HIVE4_IMAGE;
 import static java.util.Objects.requireNonNull;
 import static org.testcontainers.containers.Network.newNetwork;
 
@@ -67,7 +69,8 @@ public HiveMinIODataLake(String bucketName, Map<String, String> hiveHadoopFilesT
                 .putAll(hiveHadoopFilesToMount);
 
         String hadoopCoreSitePath = "/etc/hadoop/conf/core-site.xml";
-        if (hiveHadoopImage == HIVE3_IMAGE) {
+
+        if (Objects.equals(hiveHadoopImage, HIVE3_IMAGE) || Objects.equals(hiveHadoopImage, HIVE4_IMAGE)) {
             hadoopCoreSitePath = "/opt/hadoop/etc/hadoop/core-site.xml";
             filesToMount.put("hive_s3_insert_overwrite/hive-site.xml", "/opt/hive/conf/hive-site.xml");
         }

presto-product-tests/conf/docker/common/compose-commons.sh

@@ -26,8 +26,8 @@ function export_canonical_path() {
 
 source "${BASH_SOURCE%/*}/../../../bin/locations.sh"
 
-export DOCKER_IMAGES_VERSION=${DOCKER_IMAGES_VERSION:-10}
-export HADOOP_BASE_IMAGE=${HADOOP_BASE_IMAGE:-"prestodb/hdp2.6-hive"}
+export DOCKER_IMAGES_VERSION=${DOCKER_IMAGES_VERSION:-latest}
+export HADOOP_BASE_IMAGE=${HADOOP_BASE_IMAGE:-"imjalpreet/hdp3.1-hive"}
 # This is the directory for the overriden JDK to use for starting Presto
 export OVERRIDE_JDK_DIR=${OVERRIDE_JDK_DIR:-"/dev/null"}