Merge branch 'master' into metrics-v3-cluster-scanner

.github/workflows/replication.yaml

@@ -36,6 +36,12 @@ jobs:
  sudo sysctl net.ipv6.conf.default.disable_ipv6=0
  make test-decom
 
+ - name: Test ILM
+ run: |
+ sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+ sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+ make test-ilm
+
  - name: Test Config File
  run: |
  sudo sysctl net.ipv6.conf.all.disable_ipv6=0

Makefile

@@ -53,6 +53,10 @@ test-root-disable: install-race
  @echo "Running minio root lockdown tests"
  @env bash $(PWD)/buildscripts/disable-root.sh
 
+test-ilm: install-race
+ @echo "Running ILM tests"
+ @env bash $(PWD)/docs/bucket/replication/setup_ilm_expiry_replication.sh
+
 test-decom: install-race
  @echo "Running minio decom tests"
  @env bash $(PWD)/docs/distributed/decom.sh

cmd/bucket-replication.go

@@ -2226,7 +2226,7 @@ func getProxyTargets(ctx context.Context, bucket, object string, opts ObjectOpti
  if opts.VersionSuspended {
  return &madmin.BucketTargets{}
  }
- if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) {
+ if opts.ProxyRequest {
  return &madmin.BucketTargets{}
  }
  cfg, err := getReplicationConfig(ctx, bucket)
@@ -2247,7 +2247,7 @@ func getProxyTargets(ctx context.Context, bucket, object string, opts ObjectOpti
 func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (tgt *TargetClient, oi ObjectInfo, proxy proxyResult) {
  // this option is set when active-active replication is in place between site A -> B,
  // and site B does not have the object yet.
- if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
+ if opts.ProxyRequest { // true only when site B sets MinIOSourceProxyRequest header
  return nil, oi, proxy
  }
  var perr error
@@ -2372,7 +2372,7 @@ func scheduleReplication(ctx context.Context, oi ObjectInfo, o ObjectLayer, dsc
 func proxyTaggingToRepTarget(ctx context.Context, bucket, object string, tags *tags.Tags, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (proxy proxyResult) {
  // this option is set when active-active replication is in place between site A -> B,
  // and request hits site B that does not have the object yet.
- if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
+ if opts.ProxyRequest { // true only when site B sets MinIOSourceProxyRequest header
  return proxy
  }
  var wg sync.WaitGroup
@@ -2440,7 +2440,7 @@ func proxyTaggingToRepTarget(ctx context.Context, bucket, object string, tags *t
 func proxyGetTaggingToRepTarget(ctx context.Context, bucket, object string, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (tgs *tags.Tags, proxy proxyResult) {
  // this option is set when active-active replication is in place between site A -> B,
  // and request hits site B that does not have the object yet.
- if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
+ if opts.ProxyRequest { // true only when site B sets MinIOSourceProxyRequest header
  return nil, proxy
  }
  var wg sync.WaitGroup

cmd/ftp-server-driver.go

@@ -25,6 +25,7 @@ import (
  "fmt"
  "io"
  "os"
+ "path"
  "strings"
  "time"
 
@@ -33,6 +34,7 @@ import (
  "github.com/minio/minio-go/v7/pkg/credentials"
  "github.com/minio/minio/internal/auth"
  xioutil "github.com/minio/minio/internal/ioutil"
+ "github.com/minio/pkg/v2/mimedb"
  ftp "goftp.io/server/v2"
 )
 
@@ -103,7 +105,7 @@ type ftpMetrics struct{}
 
 var globalFtpMetrics ftpMetrics
 
-func ftpTrace(s *ftp.Context, startTime time.Time, source, path string, err error) madmin.TraceInfo {
+func ftpTrace(s *ftp.Context, startTime time.Time, source, objPath string, err error) madmin.TraceInfo {
  var errStr string
  if err != nil {
  errStr = err.Error()
@@ -114,7 +116,7 @@ func ftpTrace(s *ftp.Context, startTime time.Time, source, path string, err erro
  NodeName: globalLocalNodeName,
  FuncName: fmt.Sprintf("ftp USER=%s COMMAND=%s PARAM=%s ISLOGIN=%t, Source=%s", s.Sess.LoginUser(), s.Cmd, s.Param, s.Sess.IsLogin(), source),
  Duration: time.Since(startTime),
- Path: path,
+ Path: objPath,
  Error: errStr,
  }
 }
@@ -128,18 +130,18 @@ func (m *ftpMetrics) log(s *ftp.Context, paths ...string) func(err error) {
 }
 
 // Stat implements ftpDriver
-func (driver *ftpDriver) Stat(ctx *ftp.Context, path string) (fi os.FileInfo, err error) {
- stopFn := globalFtpMetrics.log(ctx, path)
+func (driver *ftpDriver) Stat(ctx *ftp.Context, objPath string) (fi os.FileInfo, err error) {
+ stopFn := globalFtpMetrics.log(ctx, objPath)
  defer stopFn(err)
 
- if path == SlashSeparator {
+ if objPath == SlashSeparator {
  return &minioFileInfo{
  p: SlashSeparator,
  isDir: true,
  }, nil
  }
 
- bucket, object := path2BucketObject(path)
+ bucket, object := path2BucketObject(objPath)
  if bucket == "" {
  return nil, errors.New("bucket name cannot be empty")
  }
@@ -186,8 +188,8 @@ func (driver *ftpDriver) Stat(ctx *ftp.Context, path string) (fi os.FileInfo, er
 }
 
 // ListDir implements ftpDriver
-func (driver *ftpDriver) ListDir(ctx *ftp.Context, path string, callback func(os.FileInfo) error) (err error) {
- stopFn := globalFtpMetrics.log(ctx, path)
+func (driver *ftpDriver) ListDir(ctx *ftp.Context, objPath string, callback func(os.FileInfo) error) (err error) {
+ stopFn := globalFtpMetrics.log(ctx, objPath)
  defer stopFn(err)
 
  clnt, err := driver.getMinIOClient(ctx)
@@ -198,7 +200,7 @@ func (driver *ftpDriver) ListDir(ctx *ftp.Context, path string, callback func(os
  cctx, cancel := context.WithCancel(context.Background())
  defer cancel()
 
- bucket, prefix := path2BucketObject(path)
+ bucket, prefix := path2BucketObject(objPath)
  if bucket == "" {
  buckets, err := clnt.ListBuckets(cctx)
  if err != nil {
@@ -365,11 +367,11 @@ func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error)
 }
 
 // DeleteDir implements ftpDriver
-func (driver *ftpDriver) DeleteDir(ctx *ftp.Context, path string) (err error) {
- stopFn := globalFtpMetrics.log(ctx, path)
+func (driver *ftpDriver) DeleteDir(ctx *ftp.Context, objPath string) (err error) {
+ stopFn := globalFtpMetrics.log(ctx, objPath)
  defer stopFn(err)
 
- bucket, prefix := path2BucketObject(path)
+ bucket, prefix := path2BucketObject(objPath)
  if bucket == "" {
  return errors.New("deleting all buckets not allowed")
  }
@@ -415,11 +417,11 @@ func (driver *ftpDriver) DeleteDir(ctx *ftp.Context, path string) (err error) {
 }
 
 // DeleteFile implements ftpDriver
-func (driver *ftpDriver) DeleteFile(ctx *ftp.Context, path string) (err error) {
- stopFn := globalFtpMetrics.log(ctx, path)
+func (driver *ftpDriver) DeleteFile(ctx *ftp.Context, objPath string) (err error) {
+ stopFn := globalFtpMetrics.log(ctx, objPath)
  defer stopFn(err)
 
- bucket, object := path2BucketObject(path)
+ bucket, object := path2BucketObject(objPath)
  if bucket == "" {
  return errors.New("bucket name cannot be empty")
  }
@@ -433,19 +435,19 @@ func (driver *ftpDriver) DeleteFile(ctx *ftp.Context, path string) (err error) {
 }
 
 // Rename implements ftpDriver
-func (driver *ftpDriver) Rename(ctx *ftp.Context, fromPath string, toPath string) (err error) {
- stopFn := globalFtpMetrics.log(ctx, fromPath, toPath)
+func (driver *ftpDriver) Rename(ctx *ftp.Context, fromObjPath string, toObjPath string) (err error) {
+ stopFn := globalFtpMetrics.log(ctx, fromObjPath, toObjPath)
  defer stopFn(err)
 
  return NotImplemented{}
 }
 
 // MakeDir implements ftpDriver
-func (driver *ftpDriver) MakeDir(ctx *ftp.Context, path string) (err error) {
- stopFn := globalFtpMetrics.log(ctx, path)
+func (driver *ftpDriver) MakeDir(ctx *ftp.Context, objPath string) (err error) {
+ stopFn := globalFtpMetrics.log(ctx, objPath)
  defer stopFn(err)
 
- bucket, prefix := path2BucketObject(path)
+ bucket, prefix := path2BucketObject(objPath)
  if bucket == "" {
  return errors.New("bucket name cannot be empty")
  }
@@ -461,21 +463,18 @@ func (driver *ftpDriver) MakeDir(ctx *ftp.Context, path string) (err error) {
 
  dirPath := buildMinioDir(prefix)
 
- _, err = clnt.PutObject(context.Background(), bucket, dirPath, bytes.NewReader([]byte("")), 0,
- // Always send Content-MD5 to succeed with bucket with
- // locking enabled. There is no performance hit since
- // this is always an empty object
- minio.PutObjectOptions{SendContentMd5: true},
- )
+ _, err = clnt.PutObject(context.Background(), bucket, dirPath, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{
+ DisableContentSha256: true,
+ })
  return err
 }
 
 // GetFile implements ftpDriver
-func (driver *ftpDriver) GetFile(ctx *ftp.Context, path string, offset int64) (n int64, rc io.ReadCloser, err error) {
- stopFn := globalFtpMetrics.log(ctx, path)
+func (driver *ftpDriver) GetFile(ctx *ftp.Context, objPath string, offset int64) (n int64, rc io.ReadCloser, err error) {
+ stopFn := globalFtpMetrics.log(ctx, objPath)
  defer stopFn(err)
 
- bucket, object := path2BucketObject(path)
+ bucket, object := path2BucketObject(objPath)
  if bucket == "" {
  return 0, nil, errors.New("bucket name cannot be empty")
  }
@@ -510,11 +509,11 @@ func (driver *ftpDriver) GetFile(ctx *ftp.Context, path string, offset int64) (n
 }
 
 // PutFile implements ftpDriver
-func (driver *ftpDriver) PutFile(ctx *ftp.Context, path string, data io.Reader, offset int64) (n int64, err error) {
- stopFn := globalFtpMetrics.log(ctx, path)
+func (driver *ftpDriver) PutFile(ctx *ftp.Context, objPath string, data io.Reader, offset int64) (n int64, err error) {
+ stopFn := globalFtpMetrics.log(ctx, objPath)
  defer stopFn(err)
 
- bucket, object := path2BucketObject(path)
+ bucket, object := path2BucketObject(objPath)
  if bucket == "" {
  return 0, errors.New("bucket name cannot be empty")
  }
@@ -530,8 +529,8 @@ func (driver *ftpDriver) PutFile(ctx *ftp.Context, path string, data io.Reader,
  }
 
  info, err := clnt.PutObject(context.Background(), bucket, object, data, -1, minio.PutObjectOptions{
- ContentType: "application/octet-stream",
- SendContentMd5: true,
+ ContentType:  mimedb.TypeByExtension(path.Ext(object)),
+ DisableContentSha256: true,
  })
  return info.Size, err
 }

cmd/net_test.go

@@ -131,7 +131,6 @@ func TestGetHostIP(t *testing.T) {
  expectedErr error
  }{
  {"localhost", set.CreateStringSet("127.0.0.1"), nil},
- {"example.org", set.CreateStringSet("93.184.216.34"), nil},
  }
 
  for _, testCase := range testCases {

cmd/object-api-interface.go

@@ -90,7 +90,6 @@ type ObjectOptions struct {
  PreserveETag string // preserves this etag during a PUT call.
  NoLock bool // indicates to lower layers if the caller is expecting to hold locks.
  ProxyRequest bool // only set for GET/HEAD in active-active replication scenario
- ProxyHeaderSet bool // only set for GET/HEAD in active-active replication scenario
  ReplicationRequest bool // true only if replication request
  ReplicationSourceTaggingTimestamp time.Time // set if MinIOSourceTaggingTimestamp received
  ReplicationSourceLegalholdTimestamp time.Time // set if MinIOSourceObjectLegalholdTimestamp received

cmd/object-api-options.go

@@ -66,13 +66,8 @@ func getDefaultOpts(header http.Header, copySource bool, metadata map[string]str
  if crypto.S3.IsRequested(header) || (metadata != nil && crypto.S3.IsEncrypted(metadata)) {
  opts.ServerSideEncryption = encrypt.NewSSE()
  }
- if v, ok := header[xhttp.MinIOSourceProxyRequest]; ok {
- opts.ProxyHeaderSet = true
- opts.ProxyRequest = strings.Join(v, "") == "true"
- }
- if _, ok := header[xhttp.MinIOSourceReplicationRequest]; ok {
- opts.ReplicationRequest = true
- }
+ _, opts.ProxyRequest = header[xhttp.MinIOSourceProxyRequest]
+ _, opts.ReplicationRequest = header[xhttp.MinIOSourceReplicationRequest]
  opts.Speedtest = header.Get(globalObjectPerfUserMetadata) != ""
  return
 }

cmd/sftp-server-driver.go

@@ -24,6 +24,7 @@ import (
  "fmt"
  "io"
  "os"
+ "path"
  "strings"
  "sync"
  "time"
@@ -33,6 +34,7 @@ import (
  "github.com/minio/minio-go/v7/pkg/credentials"
  "github.com/minio/minio/internal/auth"
  xioutil "github.com/minio/minio/internal/ioutil"
+ "github.com/minio/pkg/v2/mimedb"
  "github.com/pkg/sftp"
  "golang.org/x/crypto/ssh"
 )
@@ -319,7 +321,10 @@ func (f *sftpDriver) Filewrite(r *sftp.Request) (w io.WriterAt, err error) {
  }
  wa.wg.Add(1)
  go func() {
- _, err := clnt.PutObject(r.Context(), bucket, object, pr, -1, minio.PutObjectOptions{SendContentMd5: true})
+ _, err := clnt.PutObject(r.Context(), bucket, object, pr, -1, minio.PutObjectOptions{
+ ContentType: mimedb.TypeByExtension(path.Ext(object)),
+ DisableContentSha256: true,
+ })
  pr.CloseWithError(err)
  wa.wg.Done()
  }()
@@ -399,10 +404,7 @@ func (f *sftpDriver) Filecmd(r *sftp.Request) (err error) {
  dirPath := buildMinioDir(prefix)
 
  _, err = clnt.PutObject(context.Background(), bucket, dirPath, bytes.NewReader([]byte("")), 0,
- // Always send Content-MD5 to succeed with bucket with
- // locking enabled. There is no performance hit since
- // this is always an empty object
- minio.PutObjectOptions{SendContentMd5: true},
+ minio.PutObjectOptions{DisableContentSha256: true},
  )
  return err
  }

cmd/xl-storage.go

@@ -2124,14 +2124,6 @@ func (s *xlStorage) writeAllDirect(ctx context.Context, filePath string, fileSiz
 
  var bufp *[]byte
  switch {
- case fileSize > 0 && fileSize >= xioutil.XXLargeBlock*2:
- // use a larger 8MiB buffer for a really really large streamsx.
- bufp = xioutil.ODirectPoolXXLarge.Get().(*[]byte)
- defer xioutil.ODirectPoolXXLarge.Put(bufp)
- case fileSize > 0 && fileSize >= xioutil.XLargeBlock:
- // use a larger 4MiB buffer for a really large streams.
- bufp = xioutil.ODirectPoolXLarge.Get().(*[]byte)
- defer xioutil.ODirectPoolXLarge.Put(bufp)
  case fileSize <= xioutil.SmallBlock:
  bufp = xioutil.ODirectPoolSmall.Get().(*[]byte)
  defer xioutil.ODirectPoolSmall.Put(bufp)

docs/bucket/replication/setup_3site_replication.sh

@@ -166,7 +166,7 @@ echo "Set default governance retention 30d"
 ./mc retention set --default governance 30d sitea/olockbucket
 
 echo "Copying data to source sitea/bucket"
-./mc cp --encrypt "sitea/" --quiet /etc/hosts sitea/bucket
+./mc cp --enc-s3 "sitea/" --quiet /etc/hosts sitea/bucket
 sleep 1
 
 echo "Copying data to source sitea/olockbucket"
@@ -197,7 +197,7 @@ head -c 221227088 </dev/urandom >200M
 ./mc.RELEASE.2021-03-12T03-36-59Z cp --config-dir ~/.mc --encrypt "sitea" --quiet 200M "sitea/bucket/200M-enc-v1"
 ./mc.RELEASE.2021-03-12T03-36-59Z cp --config-dir ~/.mc --quiet 200M "sitea/bucket/200M-v1"
 
-./mc cp --encrypt "sitea" --quiet 200M "sitea/bucket/200M-enc-v2"
+./mc cp --enc-s3 "sitea" --quiet 200M "sitea/bucket/200M-enc-v2"
 ./mc cp --quiet 200M "sitea/bucket/200M-v2"
 
 sleep 10