Unable to read Delta tables with spaces in location or partition values

Author: denodo-research-labs

presto-delta/src/main/java/com/facebook/presto/delta/DeltaExpressionUtils.java

@@ -30,8 +30,10 @@
 import io.delta.kernel.data.Row;
 import io.delta.kernel.internal.InternalScanFileUtils;
 import io.delta.kernel.utils.CloseableIterator;
+import org.apache.hadoop.fs.Path;
 
 import java.io.IOException;
+import java.net.URI;
 import java.sql.Date;
 import java.sql.Timestamp;
 import java.util.Iterator;
@@ -239,7 +241,7 @@ private static boolean evaluatePartitionPredicate(
             for (DeltaColumnHandle partitionColumn : partitionColumns) {
                 String columnName = partitionColumn.getName();
                 String partitionValue = InternalScanFileUtils.getPartitionValues(row).get(columnName);
-                String filePath = InternalScanFileUtils.getAddFileStatus(row).getPath();
+                String filePath = new Path(URI.create(InternalScanFileUtils.getAddFileStatus(row).getPath())).toString();
                 logger.debug("Obtaining domain of file: " + filePath);
                 Domain domain = getDomain(partitionColumn, partitionValue, typeManager, filePath);
                 Optional<Map<String, Domain>> domains = partitionPredicate.getDomains();

presto-delta/src/main/java/com/facebook/presto/delta/DeltaSplitManager.java

@@ -27,9 +27,11 @@
 import io.delta.kernel.utils.CloseableIterator;
 import io.delta.kernel.utils.FileStatus;
 import jakarta.inject.Inject;
+import org.apache.hadoop.fs.Path;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.net.URI;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.concurrent.CompletableFuture;
@@ -97,7 +99,7 @@ public CompletableFuture<ConnectorSplitBatch> getNextBatch(ConnectorPartitionHan
                         connectorId,
                         deltaTable.getSchemaName(),
                         deltaTable.getTableName(),
-                        addFileStatus.getPath(),
+                        new Path(URI.create(addFileStatus.getPath())).toString(),
                         0, /* start */
                         addFileStatus.getSize() /* split length - default is read the entire file in one split */,
                         addFileStatus.getSize(),

presto-delta/src/test/java/com/facebook/presto/delta/AbstractDeltaDistributedQueryTestBase.java

@@ -62,7 +62,8 @@ public abstract class AbstractDeltaDistributedQueryTestBase
             "test-partitions-lowercase",
             "test-uppercase",
             "test-partitions-uppercase",
-            "test-typing"
+            "test-typing",
+            "test-spaces"
     };
 
     /**

presto-delta/src/test/java/com/facebook/presto/delta/TestDeltaSpaces.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.facebook.presto.delta;
+
+import org.testng.annotations.Test;
+
+import static java.lang.String.format;
+
+/**
+ * Tests for reading Delta tables with spaces in location or partition.
+ */
+@Test
+public class TestDeltaSpaces
+        extends AbstractDeltaDistributedQueryTestBase
+{
+    @Test(dataProvider = "deltaReaderVersions")
+    public void readDataWithSpaces(String version)
+    {
+        // Tests for reading Delta tables with spaces in location or partition.
+        String testQuery =
+                format("SELECT * FROM \"%s\".\"%s\" WHERE country = 'SOUTH AFRICA'", PATH_SCHEMA, goldenTablePathWithPrefix(version,
+                        "test-spaces"));
+
+        // read snapshot version 3
+        String testQueryV3 = format(testQuery, "v3");
+        String expResultsQueryV3 = "SELECT * FROM VALUES('Chima', 'manda', 'SOUTH AFRICA'), ('Adi', 'chie', 'SOUTH AFRICA')";
+        assertQuery(testQueryV3, expResultsQueryV3);
+    }
+}

presto-delta/src/test/resources/delta_v1/test-spaces/_delta_log/00000000000000000000.crc

@@ -0,0 +1 @@
+{"txnId":"0f5c61a9-c9fe-48d2-800c-24db752e063b","tableSizeBytes":0,"numFiles":0,"numDeletedRecordsOpt":0,"numDeletionVectorsOpt":0,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"72fbf1d8-14b1-4d72-acf1-4635d36eeab9","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"first_name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"last_name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"country\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["country"],"configuration":{"delta.checkpoint.writeStatsAsJson":"false","delta.checkpoint.writeStatsAsStruct":"true","delta.enableDeletionVectors":"true"},"createdTime":1756119495478},"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors","appendOnly","invariants"]},"histogramOpt":{"sortedBinBoundaries":[0,8192,16384,32768,65536,131072,262144,524288,1048576,2097152,4194304,8388608,12582912,16777216,20971520,25165824,29360128,33554432,37748736,41943040,50331648,58720256,67108864,75497472,83886080,92274688,100663296,109051904,117440512,125829120,130023424,134217728,138412032,142606336,146800640,150994944,167772160,184549376,201326592,218103808,234881024,251658240,268435456,285212672,301989888,318767104,335544320,352321536,369098752,385875968,402653184,419430400,436207616,452984832,469762048,486539264,503316480,520093696,536870912,553648128,570425344,587202560,603979776,671088640,738197504,805306368,872415232,939524096,1006632960,1073741824,1140850688,1207959552,1275068416,1342177280,1409286144,1476395008,1610612736,1744830464,1879048192,2013265920,2147483648,2415919104,2684354560,2952790016,3221225472,3489660928,3758096384,4026531840,4294967296,8589934592,17179869184,34359738368,68719476736,137438953472,274877906944],"fileCounts":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"totalBytes":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"deletedRecordCountsHistogramOpt":{"deletedRecordCounts":[0,0,0,0,0,0,0,0,0,0]},"allFiles":[]}

presto-delta/src/test/resources/delta_v1/test-spaces/_delta_log/00000000000000000000.json

@@ -0,0 +1,3 @@
+{"commitInfo":{"timestamp":1756119495831,"userId":"4986169989250709","userName":"[email protected]","operation":"CREATE TABLE","operationParameters":{"partitionBy":"[\"country\"]","clusterBy":"[]","description":null,"isManaged":"false","properties":"{\"delta.checkpoint.writeStatsAsJson\":\"false\",\"delta.checkpoint.writeStatsAsStruct\":\"true\",\"delta.enableDeletionVectors\":\"true\"}","statsOnLoad":false},"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{},"tags":{"restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/17.0.x-aarch64-photon-scala2.13","txnId":"0f5c61a9-c9fe-48d2-800c-24db752e063b"}}
+{"metaData":{"id":"72fbf1d8-14b1-4d72-acf1-4635d36eeab9","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"first_name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"last_name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"country\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["country"],"configuration":{"delta.checkpoint.writeStatsAsJson":"false","delta.checkpoint.writeStatsAsStruct":"true","delta.enableDeletionVectors":"true"},"createdTime":1756119495478}}
+{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors","appendOnly","invariants"]}}

presto-delta/src/test/resources/delta_v1/test-spaces/_delta_log/00000000000000000001.crc

@@ -0,0 +1 @@
+{"txnId":"60aa6440-4633-426a-8b4a-d45dfab4e928","tableSizeBytes":880,"numFiles":1,"numDeletedRecordsOpt":0,"numDeletionVectorsOpt":0,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"72fbf1d8-14b1-4d72-acf1-4635d36eeab9","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"first_name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"last_name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"country\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["country"],"configuration":{"delta.checkpoint.writeStatsAsJson":"false","delta.checkpoint.writeStatsAsStruct":"true","delta.enableDeletionVectors":"true"},"createdTime":1756119495478},"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors","appendOnly","invariants"]},"histogramOpt":{"sortedBinBoundaries":[0,8192,16384,32768,65536,131072,262144,524288,1048576,2097152,4194304,8388608,12582912,16777216,20971520,25165824,29360128,33554432,37748736,41943040,50331648,58720256,67108864,75497472,83886080,92274688,100663296,109051904,117440512,125829120,130023424,134217728,138412032,142606336,146800640,150994944,167772160,184549376,201326592,218103808,234881024,251658240,268435456,285212672,301989888,318767104,335544320,352321536,369098752,385875968,402653184,419430400,436207616,452984832,469762048,486539264,503316480,520093696,536870912,553648128,570425344,587202560,603979776,671088640,738197504,805306368,872415232,939524096,1006632960,1073741824,1140850688,1207959552,1275068416,1342177280,1409286144,1476395008,1610612736,1744830464,1879048192,2013265920,2147483648,2415919104,2684354560,2952790016,3221225472,3489660928,3758096384,4026531840,4294967296,8589934592,17179869184,34359738368,68719476736,137438953472,274877906944],"fileCounts":[1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"totalBytes":[880,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"deletedRecordCountsHistogramOpt":{"deletedRecordCounts":[1,0,0,0,0,0,0,0,0,0]},"allFiles":[{"path":"country=SOUTH%20AFRICA/part-00000-4aad1292-1c82-4525-a3af-4d0573f9c62d.c000.snappy.parquet","partitionValues":{"country":"SOUTH AFRICA"},"size":880,"modificationTime":1756119727000,"dataChange":false,"stats":"{\"numRecords\":1,\"minValues\":{\"first_name\":\"Chima\",\"last_name\":\"manda\"},\"maxValues\":{\"first_name\":\"Chima\",\"last_name\":\"manda\"},\"nullCount\":{\"first_name\":0,\"last_name\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1756119727000000","MIN_INSERTION_TIME":"1756119727000000","MAX_INSERTION_TIME":"1756119727000000","OPTIMIZE_TARGET_SIZE":"268435456"}}]}

presto-delta/src/test/resources/delta_v1/test-spaces/_delta_log/00000000000000000001.json

@@ -0,0 +1,2 @@
+{"commitInfo":{"timestamp":1756119727541,"userId":"4986169989250709","userName":"[email protected]","operation":"WRITE","operationParameters":{"mode":"Append","statsOnLoad":false,"partitionBy":"[]"},"readVersion":0,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"880"},"tags":{"noRowsCopied":"true","restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/17.0.x-aarch64-photon-scala2.13","txnId":"60aa6440-4633-426a-8b4a-d45dfab4e928"}}
+{"add":{"path":"country=SOUTH%20AFRICA/part-00000-4aad1292-1c82-4525-a3af-4d0573f9c62d.c000.snappy.parquet","partitionValues":{"country":"SOUTH AFRICA"},"size":880,"modificationTime":1756119727000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"first_name\":\"Chima\",\"last_name\":\"manda\"},\"maxValues\":{\"first_name\":\"Chima\",\"last_name\":\"manda\"},\"nullCount\":{\"first_name\":0,\"last_name\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1756119727000000","MIN_INSERTION_TIME":"1756119727000000","MAX_INSERTION_TIME":"1756119727000000","OPTIMIZE_TARGET_SIZE":"268435456"}}}

presto-delta/src/test/resources/delta_v1/test-spaces/_delta_log/00000000000000000002.crc

@@ -0,0 +1 @@
+{"txnId":"233a0b1b-d7d2-4b49-b47a-c3e920bf5957","tableSizeBytes":1745,"numFiles":2,"numDeletedRecordsOpt":0,"numDeletionVectorsOpt":0,"numMetadata":1,"numProtocol":1,"setTransactions":[],"domainMetadata":[],"metadata":{"id":"72fbf1d8-14b1-4d72-acf1-4635d36eeab9","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"first_name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"last_name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"country\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["country"],"configuration":{"delta.checkpoint.writeStatsAsJson":"false","delta.checkpoint.writeStatsAsStruct":"true","delta.enableDeletionVectors":"true"},"createdTime":1756119495478},"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["deletionVectors"],"writerFeatures":["deletionVectors","appendOnly","invariants"]},"histogramOpt":{"sortedBinBoundaries":[0,8192,16384,32768,65536,131072,262144,524288,1048576,2097152,4194304,8388608,12582912,16777216,20971520,25165824,29360128,33554432,37748736,41943040,50331648,58720256,67108864,75497472,83886080,92274688,100663296,109051904,117440512,125829120,130023424,134217728,138412032,142606336,146800640,150994944,167772160,184549376,201326592,218103808,234881024,251658240,268435456,285212672,301989888,318767104,335544320,352321536,369098752,385875968,402653184,419430400,436207616,452984832,469762048,486539264,503316480,520093696,536870912,553648128,570425344,587202560,603979776,671088640,738197504,805306368,872415232,939524096,1006632960,1073741824,1140850688,1207959552,1275068416,1342177280,1409286144,1476395008,1610612736,1744830464,1879048192,2013265920,2147483648,2415919104,2684354560,2952790016,3221225472,3489660928,3758096384,4026531840,4294967296,8589934592,17179869184,34359738368,68719476736,137438953472,274877906944],"fileCounts":[2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"totalBytes":[1745,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"deletedRecordCountsHistogramOpt":{"deletedRecordCounts":[2,0,0,0,0,0,0,0,0,0]},"allFiles":[{"path":"country=SOUTH%20AFRICA/part-00000-4aad1292-1c82-4525-a3af-4d0573f9c62d.c000.snappy.parquet","partitionValues":{"country":"SOUTH AFRICA"},"size":880,"modificationTime":1756119727000,"dataChange":false,"stats":"{\"numRecords\":1,\"minValues\":{\"first_name\":\"Chima\",\"last_name\":\"manda\"},\"maxValues\":{\"first_name\":\"Chima\",\"last_name\":\"manda\"},\"nullCount\":{\"first_name\":0,\"last_name\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1756119727000000","MIN_INSERTION_TIME":"1756119727000000","MAX_INSERTION_TIME":"1756119727000000","OPTIMIZE_TARGET_SIZE":"268435456"}},{"path":"country=SOUTH%20AFRICA/part-00000-e850600d-c9a8-428c-864e-308d42456618.c000.snappy.parquet","partitionValues":{"country":"SOUTH AFRICA"},"size":865,"modificationTime":1756119745000,"dataChange":false,"stats":"{\"numRecords\":1,\"minValues\":{\"first_name\":\"Adi\",\"last_name\":\"chie\"},\"maxValues\":{\"first_name\":\"Adi\",\"last_name\":\"chie\"},\"nullCount\":{\"first_name\":0,\"last_name\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1756119745000000","MIN_INSERTION_TIME":"1756119745000000","MAX_INSERTION_TIME":"1756119745000000","OPTIMIZE_TARGET_SIZE":"268435456"}}]}

presto-delta/src/test/resources/delta_v1/test-spaces/_delta_log/00000000000000000002.json

@@ -0,0 +1,2 @@
+{"commitInfo":{"timestamp":1756119745919,"userId":"4986169989250709","userName":"[email protected]","operation":"WRITE","operationParameters":{"mode":"Append","statsOnLoad":false,"partitionBy":"[]"},"readVersion":1,"isolationLevel":"WriteSerializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"1","numOutputBytes":"865"},"tags":{"noRowsCopied":"true","restoresDeletedRows":"false"},"engineInfo":"Databricks-Runtime/17.0.x-aarch64-photon-scala2.13","txnId":"233a0b1b-d7d2-4b49-b47a-c3e920bf5957"}}
+{"add":{"path":"country=SOUTH%20AFRICA/part-00000-e850600d-c9a8-428c-864e-308d42456618.c000.snappy.parquet","partitionValues":{"country":"SOUTH AFRICA"},"size":865,"modificationTime":1756119745000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"first_name\":\"Adi\",\"last_name\":\"chie\"},\"maxValues\":{\"first_name\":\"Adi\",\"last_name\":\"chie\"},\"nullCount\":{\"first_name\":0,\"last_name\":0},\"tightBounds\":true}","tags":{"INSERTION_TIME":"1756119745000000","MIN_INSERTION_TIME":"1756119745000000","MAX_INSERTION_TIME":"1756119745000000","OPTIMIZE_TARGET_SIZE":"268435456"}}}