Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cache for enterprise blobber #154

Draft
wants to merge 18 commits into
base: feat/enterprise-blobber
Choose a base branch
from
Draft
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix: list obj use prefix
harsh4723 committed Aug 19, 2024
commit c06c5e0fb17898a2d6551a79341610c4d01df75d
28 changes: 28 additions & 0 deletions cmd/bucket-listobjects-handlers.go
Original file line number Diff line number Diff line change
@@ -71,6 +71,30 @@ func mergeListObjects(l1, l2 []ObjectInfo) []ObjectInfo {
return mergedList
}

func mergePrefixes(l1, l2 []string) []string {
mergedMap := make(map[string]bool)

// Helper function to add/update map entries
addOrUpdate := func(pre string) {
if _, found := mergedMap[pre]; !found {
mergedMap[pre] = true
}
}
for _, pre := range l1 {
addOrUpdate(pre)
}
for _, pre := range l2 {
addOrUpdate(pre)
}

mergedList := make([]string, 0, len(mergedMap))
for pre, _ := range mergedMap {
mergedList = append(mergedList, pre)
}

return mergedList
}

// Validate all the ListObjects query arguments, returns an APIErrorCode
// if one of the args do not meet the required conditions.
// Special conditions required by MinIO server are as below
@@ -285,8 +309,12 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
// fmt.Printf("Harsh listObjectsV2Infooo %+v\n", listObjectsV2Info)
// fmt.Printf("Harsh listObjectsV2InfoCache %+v\n", listObjectsV2InfoCache)
mergeObjects := mergeListObjects(listObjectsV2Info.Objects, listObjectsV2InfoCache.Objects)
mergePrefixes := mergePrefixes(listObjectsV2Info.Prefixes, listObjectsV2InfoCache.Prefixes)
listObjectsV2Info.Objects = mergeObjects
listObjectsV2Info.Prefixes = mergePrefixes

concurrentDecryptETag(ctx, listObjectsV2Info.Objects)

32 changes: 30 additions & 2 deletions cmd/disk-cache.go
Original file line number Diff line number Diff line change
@@ -496,6 +496,7 @@ func (c *cacheObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dst
// ListObjectsV2 from disk cache
func (c *cacheObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
objInfos := []ObjectInfo{}
prefixes := []string{}
for _, cache := range c.cache {
dir := cache.dir
fmt.Println("cache dirss", dir)
@@ -511,8 +512,22 @@ func (c *cacheObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
return nil
}
objInfo := meta.ToObjectInfo()
if len(objInfos) < maxKeys && objInfo.Bucket == bucket {
objInfos = append(objInfos, objInfo)
if objInfo.Bucket == bucket {
if strings.HasPrefix(objInfo.Name, prefix) {
trimmed := strings.TrimPrefix(objInfo.Name, prefix)
parts := strings.Split(trimmed, delimiter)
if len(parts) > 0 && parts[0] != "" {
if (len(objInfos) + len(prefixes)) < maxKeys {
if len(parts) == 1 {
// If there's only one part, it's a file
objInfos = append(objInfos, objInfo)
} else {
// If there are more parts, it's a folder
prefixes = append(prefixes, prefix+parts[0]+delimiter)
}
}
}
}
}
return nil
}
@@ -526,9 +541,22 @@ func (c *cacheObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
return ListObjectsV2Info{
Objects: objInfos,
ContinuationToken: continuationToken,
Prefixes: unique(prefixes),
}, nil
}

func unique(items []string) []string {
keys := make(map[string]bool)
var list []string
for _, entry := range items {
if _, value := keys[entry]; !value {
keys[entry] = true
list = append(list, entry)
}
}
return list
}

// StorageInfo - returns underlying storage statistics.
func (c *cacheObjects) StorageInfo(ctx context.Context) (cInfo CacheStorageInfo) {
var total, free uint64