GODRIVER-3486 Support auto encryption in unified tests. (mongodb#2240)

Author: qingyang-hu

etc/install-libmongocrypt.sh

@@ -3,7 +3,7 @@
 # This script installs libmongocrypt into an "install" directory.
 set -eux
 
-LIBMONGOCRYPT_TAG="1.12.0"
+LIBMONGOCRYPT_TAG="1.15.1"
 
 # Install libmongocrypt based on OS.
 if [ "Windows_NT" = "${OS:-}" ]; then

internal/integration/mtest/mongotest.go

@@ -797,11 +797,6 @@ func verifyRunOnBlockConstraint(rob RunOnBlock) error {
 		return err
 	}
 
-	// TODO(GODRIVER-3486): Once auto encryption is supported by the unified test
-	// format,this check should be removed.
-	if rob.CSFLEEnabled() && rob.CSFLE.Options != nil {
-		return fmt.Errorf("auto encryption required (GODRIVER-3486)")
-	}
 	if rob.CSFLEEnabled() && !IsCSFLEEnabled() {
 		return fmt.Errorf("runOnBlock requires CSFLE to be enabled. Build with the cse tag to enable")
 	}

internal/integration/mtest/options.go

@@ -51,7 +51,9 @@ var (
 
 // CSFLEOptions holds configuration for Client-Side Field Level Encryption
 // (CSFLE).
-type CSFLEOptions struct{}
+type CSFLEOptions struct {
+	MinVer string `bson:"minLibmongocryptVersion"`
+}
 
 // CSFLE models the runOnRequirements.csfle field in Unified Test Format tests.
 //
@@ -67,12 +69,12 @@ type CSFLE struct {
 
 // UnmarshalBSON implements custom BSON unmarshalling for CSFLE, accepting
 // either a boolean or an embedded document. If a document is provided, Options
-// is set and Boolean is false. If a boolean is provided, Boolean is set and
+// is set and Boolean is true. If a boolean is provided, Boolean is set and
 // Options is nil.
 func (csfle *CSFLE) UnmarshalBSON(data []byte) error {
 	embRawValue := bson.RawValue{Type: bson.TypeEmbeddedDocument, Value: data}
 	if err := embRawValue.Unmarshal(&csfle.Options); err == nil {
-		csfle.Boolean = false
+		csfle.Boolean = true
 
 		return nil
 	}

internal/integration/unified/client_entity.go

@@ -8,6 +8,7 @@ package unified
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"strings"
 	"sync"
@@ -198,6 +199,13 @@ func newClientEntity(ctx context.Context, em *EntityMap, entityOptions *entityOp
 	} else {
 		integtest.AddTestServerAPIVersion(clientOpts)
 	}
+	if entityOptions.AutoEncryptOpts != nil {
+		aeo, err := createAutoEncryptionOptions(entityOptions.AutoEncryptOpts)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing auto encryption options: %w", err)
+		}
+		clientOpts.SetAutoEncryptionOptions(aeo)
+	}
 	for _, cmd := range entityOptions.IgnoredCommands {
 		entity.ignoredCommands[cmd] = struct{}{}
 	}
@@ -241,6 +249,83 @@ func getURIForClient(opts *entityOptions) string {
 	}
 }
 
+// TODO(GODRIVER-3726): Need to update the logic to an Unmarshal method.
+func createAutoEncryptionOptions(opts bson.Raw) (*options.AutoEncryptionOptions, error) {
+	aeo := options.AutoEncryption()
+	var kvnsFound bool
+	elems, err := opts.Elements()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, elem := range elems {
+		name := elem.Key()
+		opt := elem.Value()
+
+		switch name {
+		case "kmsProviders":
+			providers := make(map[string]map[string]any)
+			elems, err := opt.Document().Elements()
+			if err != nil {
+				return nil, err
+			}
+			for _, elem := range elems {
+				key := elem.Key()
+				opt := elem.Value().Document()
+				provider, err := getKmsProvider(key, opt)
+				if err != nil {
+					return nil, err
+				}
+				if provider == nil {
+					continue
+				}
+				providers[key] = provider
+				if key == "kmip" && tlsClientCertificateKeyFile != "" && tlsCAFile != "" {
+					cfg, err := options.BuildTLSConfig(map[string]any{
+						"tlsCertificateKeyFile": tlsClientCertificateKeyFile,
+						"tlsCAFile":             tlsCAFile,
+					})
+					if err != nil {
+						return nil, fmt.Errorf("error constructing tls config: %w", err)
+					}
+					aeo.SetTLSConfig(map[string]*tls.Config{
+						"kmip": cfg,
+					})
+				}
+			}
+			aeo.SetKmsProviders(providers)
+		case "schemaMap":
+			var schemaMap map[string]any
+			err := bson.Unmarshal(opt.Document(), &schemaMap)
+			if err != nil {
+				return nil, fmt.Errorf("error creating schema map: %v", err)
+			}
+			aeo.SetSchemaMap(schemaMap)
+		case "keyVaultNamespace":
+			kvnsFound = true
+			aeo.SetKeyVaultNamespace(opt.StringValue())
+		case "bypassAutoEncryption":
+			aeo.SetBypassAutoEncryption(opt.Boolean())
+		case "encryptedFieldsMap":
+			var encryptedFieldsMap map[string]any
+			err := bson.Unmarshal(opt.Document(), &encryptedFieldsMap)
+			if err != nil {
+				return nil, fmt.Errorf("error creating encryptedFieldsMap: %v", err)
+			}
+			aeo.SetEncryptedFieldsMap(encryptedFieldsMap)
+		case "bypassQueryAnalysis":
+			aeo.SetBypassQueryAnalysis(opt.Boolean())
+		default:
+			return nil, fmt.Errorf("unrecognized option: %v", name)
+		}
+	}
+	if !kvnsFound {
+		aeo.SetKeyVaultNamespace("keyvault.datakeys")
+	}
+
+	return aeo, nil
+}
+
 // disconnect disconnects the client associated with this entity. It is an
 // idempotent operation, unlike the mongo client's disconnect method. This
 // property will help avoid unnecessary errors when calling disconnect on a

internal/integration/unified/collection_data.go

@@ -27,8 +27,10 @@ type collectionData struct {
 }
 
 type createOptions struct {
-	Capped      *bool  `bson:"capped"`
-	SizeInBytes *int64 `bson:"size"`
+	Capped          *bool    `bson:"capped"`
+	SizeInBytes     *int64   `bson:"size"`
+	EncryptedFields bson.Raw `bson:"encryptedFields"`
+	Validator       bson.Raw `bson:"validator"`
 }
 
 // createCollection configures the collection represented by the receiver using the internal client. This function
@@ -49,14 +51,18 @@ func (c *collectionData) createCollection(ctx context.Context) error {
 		if c.Options.SizeInBytes != nil {
 			createOpts = createOpts.SetSizeInBytes(*c.Options.SizeInBytes)
 		}
+		if c.Options.EncryptedFields != nil {
+			createOpts = createOpts.SetEncryptedFields(c.Options.EncryptedFields)
+		}
+		if c.Options.Validator != nil {
+			createOpts = createOpts.SetValidator(c.Options.Validator)
+		}
 
 		if err := db.CreateCollection(ctx, c.CollectionName, createOpts); err != nil {
 			return fmt.Errorf("error creating collection: %w", err)
 		}
-	}
-
-	// If neither documents nor options are provided, still create the collection with write concern "majority".
-	if len(c.Documents) == 0 && c.Options == nil {
+	} else {
+		// If no options are provided, still create the collection with write concern "majority".
 		// The write concern has to be manually specified in the command document because RunCommand does not honor
 		// the database's write concern.
 		create := bson.D{
@@ -68,13 +74,15 @@ func (c *collectionData) createCollection(ctx context.Context) error {
 		if err := db.RunCommand(ctx, create).Err(); err != nil {
 			return fmt.Errorf("error creating collection: %w", err)
 		}
-		return nil
 	}
 
-	docs := bsonutil.RawToInterfaces(c.Documents...)
-	if _, err := coll.InsertMany(ctx, docs); err != nil {
-		return fmt.Errorf("error inserting data: %w", err)
+	if len(c.Documents) != 0 {
+		docs := bsonutil.RawToInterfaces(c.Documents...)
+		if _, err := coll.InsertMany(ctx, docs); err != nil {
+			return fmt.Errorf("error inserting data: %w", err)
+		}
 	}
+
 	return nil
 }
 

internal/integration/unified/database_operation_execution.go

@@ -125,6 +125,10 @@ func executeCreateCollection(ctx context.Context, operation *operation) (*operat
 			cco.SetTimeSeriesOptions(tso)
 		case "clusteredIndex":
 			cco.SetClusteredIndex(val.Document())
+		case "validator":
+			cco.SetValidator(val.Document())
+		case "encryptedFields":
+			cco.SetEncryptedFields(val.Document())
 		default:
 			return nil, fmt.Errorf("unrecognized createCollection option %q", key)
 		}
@@ -156,13 +160,17 @@ func executeDropCollection(ctx context.Context, operation *operation) (*operatio
 		return nil, err
 	}
 
+	dco := options.DropCollection()
+
 	var collName string
 	elems, _ := operation.Arguments.Elements()
 	for _, elem := range elems {
 		key := elem.Key()
 		val := elem.Value()
 
 		switch key {
+		case "encryptedFields":
+			dco.SetEncryptedFields(val.Document())
 		case "collection":
 			collName = val.StringValue()
 		default:
@@ -173,7 +181,7 @@ func executeDropCollection(ctx context.Context, operation *operation) (*operatio
 		return nil, newMissingArgumentError("collection")
 	}
 
-	err = db.Collection(collName).Drop(ctx)
+	err = db.Collection(collName).Drop(ctx, dco)
 	return newErrorResult(err), nil
 }