feat: use env var to enable search index

0chain · Oct 31, 2024 · b271d02 · b271d02
1 parent 3eefa1b
commit b271d02
Show file tree

Hide file tree

Showing 4 changed files with 53 additions and 44 deletions.
diff --git a/cmd/disk-cache.go b/cmd/disk-cache.go
@@ -114,12 +114,13 @@ type cacheObjects struct {
 	// number of accesses after which to cache an object
 	after int
 	// commit objects in async manner
-	commitWriteback    bool
-	commitWritethrough bool
-	maxCacheFileSize   int64
-	uploadWorkers      int
-	uploadQueueTh      int
-	indexSvcUrl        string
+	commitWriteback     bool
+	commitWritethrough  bool
+	maxCacheFileSize    int64
+	uploadWorkers       int
+	uploadQueueTh       int
+	indexSvcUrl         string
+	contentSearchEnable string
 	// if true migration is in progress from v1 to v2
 	migrating bool
 	// retry queue for writeback cache mode to reattempt upload to backend
@@ -985,14 +986,15 @@ func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
 		time.Sleep(time.Second * time.Duration(retryCnt%10+1))
 		c.queueWritebackRetry(oi)
 	}
-
-	log.Println("indexing file started")
-	cReader2, _, bErr2 := dcache.Get(ctx, oi.Bucket, oi.Name, nil, http.Header{}, ObjectOptions{})
-	if bErr2 != nil {
-		return
+	if c.contentSearchEnable == "true" {
+		log.Println("indexing file started")
+		cReader2, _, bErr2 := dcache.Get(ctx, oi.Bucket, oi.Name, nil, http.Header{}, ObjectOptions{})
+		if bErr2 != nil {
+			return
+		}
+		defer cReader2.Close()
+		c.indexFile(cReader2, oi.Bucket, oi.Name)
 	}
-	defer cReader2.Close()
-	c.indexFile(cReader2, oi.Bucket, oi.Name)
 }
 
 func (c *cacheObjects) indexFile(body io.ReadCloser, bucket string, object string) {
@@ -1061,6 +1063,7 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 		uploadWorkers:           config.UploadWorkers,
 		uploadQueueTh:           config.UploadQueueTh,
 		indexSvcUrl:             config.IndexSvcUrl,
+		contentSearchEnable:     config.ContentSearchEnable,
 		cacheStats:              newCacheStats(),
 		listTree:                newThreadSafeListTree(),
 		writeBackUploadBufferCh: make(chan ObjectInfo, 100000),

diff --git a/environment/docker-compose.yaml b/environment/docker-compose.yaml
@@ -55,6 +55,7 @@ services:
       MINIO_WRITE_BACK_UPLOAD_WORKERS: 20
       MINIO_UPLOAD_QUEUE_TH: 10
       INDEX_SVC_URL: "http://zsearch:3003"
+      CONTENT_SEARCH_ENABLE: "true"
     links:
       - logsearchapi:logsearchapi
     volumes:

diff --git a/internal/config/cache/config.go b/internal/config/cache/config.go
@@ -36,22 +36,23 @@ const (
 
 // Config represents cache config settings
 type Config struct {
-	Enabled           bool     `json:"-"`
-	Drives            []string `json:"drives"`
-	Expiry            int      `json:"expiry"`
-	MaxUse            int      `json:"maxuse"`
-	Quota             int      `json:"quota"`
-	Exclude           []string `json:"exclude"`
-	After             int      `json:"after"`
-	WatermarkLow      int      `json:"watermark_low"`
-	WatermarkHigh     int      `json:"watermark_high"`
-	Range             bool     `json:"range"`
-	CacheCommitMode   string   `json:"commit"`
-	WriteBackInterval int      `json:"wb_interval"`
-	MaxCacheFileSize  int64    `json:"max_cache_file_size"`
-	UploadWorkers     int      `json:"upload_workers"`
-	UploadQueueTh     int      `json:"upload_queue_th"`
-	IndexSvcUrl       string   `json:"index_svc_url"`
+	Enabled             bool     `json:"-"`
+	Drives              []string `json:"drives"`
+	Expiry              int      `json:"expiry"`
+	MaxUse              int      `json:"maxuse"`
+	Quota               int      `json:"quota"`
+	Exclude             []string `json:"exclude"`
+	After               int      `json:"after"`
+	WatermarkLow        int      `json:"watermark_low"`
+	WatermarkHigh       int      `json:"watermark_high"`
+	Range               bool     `json:"range"`
+	CacheCommitMode     string   `json:"commit"`
+	WriteBackInterval   int      `json:"wb_interval"`
+	MaxCacheFileSize    int64    `json:"max_cache_file_size"`
+	UploadWorkers       int      `json:"upload_workers"`
+	UploadQueueTh       int      `json:"upload_queue_th"`
+	IndexSvcUrl         string   `json:"index_svc_url"`
+	ContentSearchEnable string   `json:"content_search_enable"`
 }
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling

diff --git a/internal/config/cache/lookup.go b/internal/config/cache/lookup.go
@@ -38,21 +38,22 @@ const (
 	Range         = "range"
 	Commit        = "commit"
 
-	EnvCacheDrives        = "MINIO_CACHE_DRIVES"
-	EnvCacheExclude       = "MINIO_CACHE_EXCLUDE"
-	EnvCacheExpiry        = "MINIO_CACHE_EXPIRY"
-	EnvCacheMaxUse        = "MINIO_CACHE_MAXUSE"
-	EnvCacheQuota         = "MINIO_CACHE_QUOTA"
-	EnvCacheAfter         = "MINIO_CACHE_AFTER"
-	EnvCacheWatermarkLow  = "MINIO_CACHE_WATERMARK_LOW"
-	EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
-	EnvCacheRange         = "MINIO_CACHE_RANGE"
-	EnvCacheCommit        = "MINIO_CACHE_COMMIT"
-	EnvWriteBackInterval  = "MINIO_WRITE_BACK_INTERVAL"
-	EnvMaxCacheFileSize   = "MINIO_MAX_CACHE_FILE_SIZE"
-	EnvUploadWorkers      = "MINIO_WRITE_BACK_UPLOAD_WORKERS"
-	EnvUploadQueueTh      = "MINIO_UPLOAD_QUEUE_TH"
-	EnvIndexSvcUrl        = "INDEX_SVC_URL"
+	EnvCacheDrives         = "MINIO_CACHE_DRIVES"
+	EnvCacheExclude        = "MINIO_CACHE_EXCLUDE"
+	EnvCacheExpiry         = "MINIO_CACHE_EXPIRY"
+	EnvCacheMaxUse         = "MINIO_CACHE_MAXUSE"
+	EnvCacheQuota          = "MINIO_CACHE_QUOTA"
+	EnvCacheAfter          = "MINIO_CACHE_AFTER"
+	EnvCacheWatermarkLow   = "MINIO_CACHE_WATERMARK_LOW"
+	EnvCacheWatermarkHigh  = "MINIO_CACHE_WATERMARK_HIGH"
+	EnvCacheRange          = "MINIO_CACHE_RANGE"
+	EnvCacheCommit         = "MINIO_CACHE_COMMIT"
+	EnvWriteBackInterval   = "MINIO_WRITE_BACK_INTERVAL"
+	EnvMaxCacheFileSize    = "MINIO_MAX_CACHE_FILE_SIZE"
+	EnvUploadWorkers       = "MINIO_WRITE_BACK_UPLOAD_WORKERS"
+	EnvUploadQueueTh       = "MINIO_UPLOAD_QUEUE_TH"
+	EnvIndexSvcUrl         = "INDEX_SVC_URL"
+	EnvContentSearchEnable = "CONTENT_SEARCH_ENABLE"
 
 	EnvCacheEncryptionKey = "MINIO_CACHE_ENCRYPTION_SECRET_KEY"
 
@@ -265,5 +266,8 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 	if indexSvcUrl := env.Get(EnvIndexSvcUrl, "http://zsearch:3003"); indexSvcUrl != "" {
 		cfg.IndexSvcUrl = indexSvcUrl
 	}
+	if contentSearchEnable := env.Get(EnvContentSearchEnable, "false"); contentSearchEnable != "" {
+		cfg.ContentSearchEnable = contentSearchEnable
+	}
 	return cfg, nil
 }