Add delimiter support for S3 List API

ambry-api/src/main/java/com/github/ambry/config/FrontendConfig.java

@@ -22,6 +22,8 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
+import static com.github.ambry.rest.RestUtils.*;
+
 
 /**
  * Configuration parameters required by the Ambry frontend.
@@ -44,7 +46,9 @@ public class FrontendConfig {
   public static final String CONTAINER_METRICS_AGGREGATED_ACCOUNTS = PREFIX + "container.metrics.aggregated.accounts";
   public static final String ACCOUNT_STATS_STORE_FACTORY = PREFIX + "account.stats.store.factory";
   public static final String CONTAINER_METRICS_ENABLED_REQUEST_TYPES = PREFIX + "container.metrics.enabled.request.types";
-  public static final String CONTAINER_METRICS_ENABLED_GET_REQUEST_TYPES = PREFIX + "container.metrics.enabled.get.request.types";
+  public static final String CONTAINER_METRICS_ENABLED_GET_REQUEST_TYPES =
+      PREFIX + "container.metrics.enabled.get.request.types";
+  public static final String LIST_MAX_RESULTS = PREFIX + "list.max.results";
 
   // Default values
   private static final String DEFAULT_ENDPOINT = "http://localhost:1174";
@@ -294,6 +298,14 @@ public class FrontendConfig {
    */
   public final boolean oneHundredContinueEnable;
 
+  /**
+   * The maximum number of entries to return per response page when listing blobs.
+   * TODO: Remove the config in {@link MySqlNamedBlobDbConfig} later.
+   */
+  @Config(LIST_MAX_RESULTS)
+  @Default("1000")
+  public final int listMaxResults;
+
   public FrontendConfig(VerifiableProperties verifiableProperties) {
     NettyConfig nettyConfig = new NettyConfig(verifiableProperties);
     cacheValiditySeconds = verifiableProperties.getLong("frontend.cache.validity.seconds", 365 * 24 * 60 * 60);
@@ -357,6 +369,8 @@ public FrontendConfig(VerifiableProperties verifiableProperties) {
         Utils.splitString(verifiableProperties.getString(CONTAINER_METRICS_EXCLUDED_ACCOUNTS, ""), ",");
     containerMetricsAggregatedAccounts =
         Utils.splitString(verifiableProperties.getString(CONTAINER_METRICS_AGGREGATED_ACCOUNTS, ""), ",");
+    this.listMaxResults =
+        verifiableProperties.getIntInRange(LIST_MAX_RESULTS, DEFAULT_MAX_KEY_VALUE, 1, Integer.MAX_VALUE);
   }
 
   /**

ambry-api/src/main/java/com/github/ambry/frontend/NamedBlobListEntry.java

@@ -29,40 +29,47 @@ public class NamedBlobListEntry {
   private static final String EXPIRATION_TIME_MS_KEY = "expirationTimeMs";
   private static final String BLOB_SIZE_KEY = "blobSize";
   private static final String MODIFIED_TIME_MS_KEY = "modifiedTimeMs";
+  private static final String IS_DIRECTORY_KEY = "isDirectory";
 
   private final String blobName;
   private final long expirationTimeMs;
   private final long blobSize;
   private final long modifiedTimeMs;
+  private final boolean isDirectory;
 
   /**
    * Read a {@link NamedBlobRecord} from JSON.
    * @param jsonObject the {@link JSONObject} to deserialize.
    */
   public NamedBlobListEntry(JSONObject jsonObject) {
     this(jsonObject.getString(BLOB_NAME_KEY), jsonObject.optLong(EXPIRATION_TIME_MS_KEY, Utils.Infinite_Time),
-        jsonObject.optLong(BLOB_SIZE_KEY, 0), jsonObject.optLong(MODIFIED_TIME_MS_KEY, Utils.Infinite_Time));
+        jsonObject.optLong(BLOB_SIZE_KEY, 0), jsonObject.optLong(MODIFIED_TIME_MS_KEY, Utils.Infinite_Time),
+        jsonObject.optBoolean(IS_DIRECTORY_KEY, false));
   }
 
   /**
    * Convert a {@link NamedBlobRecord} into a {@link NamedBlobListEntry}.
    * @param record the {@link NamedBlobRecord}.
    */
   NamedBlobListEntry(NamedBlobRecord record) {
-    this(record.getBlobName(), record.getExpirationTimeMs(), record.getBlobSize(), record.getModifiedTimeMs());
+    this(record.getBlobName(), record.getExpirationTimeMs(), record.getBlobSize(), record.getModifiedTimeMs(),
+        record.isDirectory());
   }
 
   /**
-   * @param blobName the blob name within a container.
+   * @param blobName         the blob name within a container.
    * @param expirationTimeMs the expiration time in milliseconds since epoch, or -1 if the blob should be permanent.
    * @param blobSize         the size of the blob
    * @param modifiedTimeMs   the modified time of the blob in milliseconds since epoch
+   * @param isDirectory      whether the blob is a directory (virtual folder name separated by '/')
    */
-  private NamedBlobListEntry(String blobName, long expirationTimeMs, long blobSize, long modifiedTimeMs) {
+  private NamedBlobListEntry(String blobName, long expirationTimeMs, long blobSize, long modifiedTimeMs,
+      boolean isDirectory) {
     this.blobName = blobName;
     this.expirationTimeMs = expirationTimeMs;
     this.blobSize = blobSize;
     this.modifiedTimeMs = modifiedTimeMs;
+    this.isDirectory = isDirectory;
   }
 
   /**
@@ -103,6 +110,7 @@ public JSONObject toJson() {
     }
     jsonObject.put(BLOB_SIZE_KEY, blobSize);
     jsonObject.put(MODIFIED_TIME_MS_KEY, modifiedTimeMs);
+    jsonObject.put(IS_DIRECTORY_KEY, isDirectory);
     return jsonObject;
   }
 
@@ -116,6 +124,10 @@ public boolean equals(Object o) {
     }
     NamedBlobListEntry that = (NamedBlobListEntry) o;
     return expirationTimeMs == that.expirationTimeMs && Objects.equals(blobName, that.blobName)
-        && modifiedTimeMs == that.modifiedTimeMs && blobSize == that.blobSize;
+        && modifiedTimeMs == that.modifiedTimeMs && blobSize == that.blobSize && isDirectory == that.isDirectory;
+  }
+
+  public boolean isDirectory() {
+    return isDirectory;
   }
 }

ambry-api/src/main/java/com/github/ambry/frontend/s3/S3MessagePayload.java

@@ -14,12 +14,10 @@
  */
 package com.github.ambry.frontend.s3;
 
-import java.util.ArrayList;
-import java.util.List;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
-import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
-import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.List;
 
 
 /**
@@ -157,6 +155,7 @@ public String toString() {
     }
   }
 
+  @JsonInclude(JsonInclude.Include.NON_EMPTY)
   public static abstract class AbstractListBucketResult {
     @JacksonXmlProperty(localName = "Name")
     private String name;
@@ -175,13 +174,17 @@ public static abstract class AbstractListBucketResult {
     private String encodingType;
     @JacksonXmlProperty(localName = "IsTruncated")
     private Boolean isTruncated;
+    // New optional field for CommonPrefixes with wrapping and nested Prefix elements
+    @JacksonXmlProperty(localName = "CommonPrefixes")
+    @JacksonXmlElementWrapper(useWrapping = false)
+    private List<Prefix> commonPrefixes;
 
     private AbstractListBucketResult() {
 
     }
 
     public AbstractListBucketResult(String name, String prefix, int maxKeys, int keyCount, String delimiter,
-        List<Contents> contents, String encodingType, boolean isTruncated) {
+        List<Contents> contents, String encodingType, boolean isTruncated, List<Prefix> commonPrefixes) {
       this.name = name;
       this.prefix = prefix;
       this.maxKeys = maxKeys;
@@ -190,6 +193,7 @@ public AbstractListBucketResult(String name, String prefix, int maxKeys, int key
       this.contents = contents;
       this.encodingType = encodingType;
       this.isTruncated = isTruncated;
+      this.commonPrefixes = commonPrefixes;
     }
 
     public String getPrefix() {
@@ -227,13 +231,23 @@ public String getName() {
     @Override
     public String toString() {
       return "Name=" + name + ", Prefix=" + prefix + ", MaxKeys=" + maxKeys + ", KeyCount=" + keyCount + ", Delimiter="
-          + delimiter + ", Contents=" + contents + ", Encoding type=" + encodingType + ", IsTruncated=" + isTruncated;
+          + delimiter + ", Contents=" + contents + ", Encoding type=" + encodingType + ", IsTruncated=" + isTruncated
+          + ", CommonPrefixes=" + commonPrefixes;
+    }
+
+    public List<Prefix> getCommonPrefixes() {
+      return commonPrefixes;
+    }
+
+    public void setCommonPrefixes(List<Prefix> commonPrefixes) {
+      this.commonPrefixes = commonPrefixes;
     }
   }
 
   /**
    * ListBucketResult for listObjects API.
    */
+  @JsonInclude(JsonInclude.Include.NON_EMPTY)
   public static class ListBucketResult extends AbstractListBucketResult {
     @JacksonXmlProperty(localName = "Marker")
     private String marker;
@@ -245,8 +259,9 @@ private ListBucketResult() {
     }
 
     public ListBucketResult(String name, String prefix, int maxKeys, int keyCount, String delimiter,
-        List<Contents> contents, String encodingType, String marker, String nextMarker, boolean isTruncated) {
-      super(name, prefix, maxKeys, keyCount, delimiter, contents, encodingType, isTruncated);
+        List<Contents> contents, String encodingType, String marker, String nextMarker, boolean isTruncated,
+        List<Prefix> commonPrefixes) {
+      super(name, prefix, maxKeys, keyCount, delimiter, contents, encodingType, isTruncated, commonPrefixes);
       this.marker = marker;
       this.nextMarker = nextMarker;
     }
@@ -268,6 +283,7 @@ public String toString() {
   /**
    * ListBucketResult for listObjectsV2 API.
    */
+  @JsonInclude(JsonInclude.Include.NON_EMPTY)
   public static class ListBucketResultV2 extends AbstractListBucketResult {
     @JacksonXmlProperty(localName = "ContinuationToken")
     private String continuationToken;
@@ -280,8 +296,8 @@ private ListBucketResultV2() {
 
     public ListBucketResultV2(String name, String prefix, int maxKeys, int keyCount, String delimiter,
         List<Contents> contents, String encodingType, String continuationToken, String nextContinuationToken,
-        boolean isTruncated) {
-      super(name, prefix, maxKeys, keyCount, delimiter, contents, encodingType, isTruncated);
+        boolean isTruncated, List<Prefix> commonPrefixes) {
+      super(name, prefix, maxKeys, keyCount, delimiter, contents, encodingType, isTruncated, commonPrefixes);
       this.continuationToken = continuationToken;
       this.nextContinuationToken = nextContinuationToken;
     }
@@ -322,7 +338,9 @@ public String getKey() {
       return key;
     }
 
-    public long getSize() { return size; }
+    public long getSize() {
+      return size;
+    }
 
     public String getLastModified() {
       return lastModified;
@@ -359,11 +377,38 @@ public String toString() {
     }
   }
 
+  // Inner class for wrapping each Prefix inside CommmomPrefixes
+  public static class Prefix {
+    @JacksonXmlProperty(localName = "Prefix")
+    private String prefix;
+
+    public Prefix() {
+    }
+
+    public Prefix(String prefix) {
+      this.prefix = prefix;
+    }
+
+    public String getPrefix() {
+      return prefix;
+    }
+
+    public void setPrefix(String prefix) {
+      this.prefix = prefix;
+    }
+
+    @Override
+    public String toString() {
+      return "Prefix=" + prefix;
+    }
+  }
+
   public static class S3BatchDeleteObjects {
 
     // Ensure that the "Delete" wrapper element is mapped correctly to the list of "Object" elements
     @JacksonXmlElementWrapper(useWrapping = false)  // Avoids wrapping the <Delete> element itself
-    @JacksonXmlProperty(localName = "Object")      // Specifies that each <Object> element maps to an instance of S3BatchDeleteKeys
+    @JacksonXmlProperty(localName = "Object")
+    // Specifies that each <Object> element maps to an instance of S3BatchDeleteKeys
     private List<S3BatchDeleteKey> objects;
 
     public List<S3BatchDeleteKey> getObjects() {

ambry-api/src/main/java/com/github/ambry/named/NamedBlobDb.java

@@ -60,8 +60,8 @@ default CompletableFuture<NamedBlobRecord> get(String accountName, String contai
    * @param pageToken      if {@code null}, return the first page of {@link NamedBlobRecord}s that start with
    *                       {@code blobNamePrefix}. If set, use this as a token to resume reading additional pages of
    *                       records that start with the prefix.
-   * @param maxKey the maximum number of keys returned in the response. By default, the action returns up to listMaxResults
-   *               which can be tuned by config.
+   * @param maxKey         the maximum number of keys returned in the response. By default, the action returns up to
+   *                       listMaxResults which can be tuned by config.
    * @return a {@link CompletableFuture} that will eventually contain a {@link Page} of {@link NamedBlobRecord}s
    * starting with the specified prefix or an exception if an error occurred.
    */

ambry-api/src/main/java/com/github/ambry/named/NamedBlobRecord.java

@@ -30,6 +30,7 @@ public class NamedBlobRecord {
   private final String blobId;
   private final long blobSize;
   private long modifiedTimeMs;
+  private final boolean isDirectory;
 
   /**
    * @param accountName the account name.
@@ -67,7 +68,7 @@ public NamedBlobRecord(String accountName, String containerName, String blobName
    */
   public NamedBlobRecord(String accountName, String containerName, String blobName, String blobId,
       long expirationTimeMs, long version, long blobSize) {
-    this(accountName, containerName, blobName, blobId, expirationTimeMs, version, blobSize, 0);
+    this(accountName, containerName, blobName, blobId, expirationTimeMs, version, blobSize, 0, false);
   }
 
   /**
@@ -79,9 +80,10 @@ public NamedBlobRecord(String accountName, String containerName, String blobName
    * @param version          the version of this named blob.
    * @param blobSize         the size of the blob.
    * @param modifiedTimeMs   the modified time of the blob in milliseconds since epoch
+   * @param isDirectory      whether the blob is a directory (virtual folder name separated by '/')
    */
   public NamedBlobRecord(String accountName, String containerName, String blobName, String blobId,
-      long expirationTimeMs, long version, long blobSize, long modifiedTimeMs) {
+      long expirationTimeMs, long version, long blobSize, long modifiedTimeMs, boolean isDirectory) {
     this.accountName = accountName;
     this.containerName = containerName;
     this.blobName = blobName;
@@ -90,6 +92,7 @@ public NamedBlobRecord(String accountName, String containerName, String blobName
     this.version = version;
     this.blobSize = blobSize;
     this.modifiedTimeMs = modifiedTimeMs;
+    this.isDirectory = isDirectory;
   }
 
   /**
@@ -180,4 +183,11 @@ public long getModifiedTimeMs() {
   public void setModifiedTimeMs(long modifiedTimeMs) {
     this.modifiedTimeMs = modifiedTimeMs;
   }
+
+  /**
+   * @return whether the blob is a directory (virtual folder name separated by '/')
+   */
+  public boolean isDirectory() {
+    return isDirectory;
+  }
 }

ambry-frontend/src/integration-test/java/com/github/ambry/frontend/FrontendIntegrationTest.java

@@ -739,6 +739,7 @@ private static VerifiableProperties buildFrontendVProps(File trustStoreFile, boo
     properties.setProperty(FrontendConfig.ENABLE_UNDELETE, Boolean.toString(enableUndelete));
     properties.setProperty(FrontendConfig.NAMED_BLOB_DB_FACTORY, "com.github.ambry.frontend.TestNamedBlobDbFactory");
     properties.setProperty(MySqlNamedBlobDbConfig.LIST_MAX_RESULTS, String.valueOf(NAMED_BLOB_LIST_RESULT_MAX));
+    properties.setProperty(FrontendConfig.LIST_MAX_RESULTS, String.valueOf(NAMED_BLOB_LIST_RESULT_MAX));
     return new VerifiableProperties(properties);
   }
 

ambry-frontend/src/integration-test/java/com/github/ambry/frontend/FrontendIntegrationTestBase.java

@@ -13,6 +13,7 @@
  */
 package com.github.ambry.frontend;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.github.ambry.account.Account;
 import com.github.ambry.account.AccountCollectionSerde;
@@ -1847,15 +1848,18 @@ static class NamedBlobEntry {
     private long expirationTimeMs;
     private long blobSize;
     private long modifiedTimeMs;
+    @JsonProperty("isDirectory")
+    private boolean isDirectory;
 
     public NamedBlobEntry() {
     }
 
-    public NamedBlobEntry(String blobName, long expiration, long blobSize, long modifiedTimeMs) {
+    public NamedBlobEntry(String blobName, long expiration, long blobSize, long modifiedTimeMs, boolean isDirectory) {
       this.blobName = blobName;
       this.expirationTimeMs = expiration;
       this.blobSize = blobSize;
       this.modifiedTimeMs = modifiedTimeMs;
+      this.isDirectory = isDirectory;
     }
 
     public String getBlobName() {
@@ -1889,6 +1893,14 @@ public long getModifiedTimeMs() {
     public void setModifiedTimeMs(long modifiedTimeMs) {
       this.modifiedTimeMs = modifiedTimeMs;
     }
+
+    public boolean isDirectory() {
+      return isDirectory;
+    }
+
+    public void setDirectory(boolean isDirectory) {
+      this.isDirectory = isDirectory;
+    }
   }
 
   /**