Merge pull request #15139 from mongodb-js/NODE-6505/ci-setup

ci(NODE-6505): CI Setup for Encryption Support

Author: vkarpov15

.eslintrc.js

@@ -14,7 +14,8 @@ module.exports = {
     '**/docs/js/native.js',
     '!.*',
     'node_modules',
-    '.git'
+    '.git',
+    'data'
   ],
   overrides: [
     {

.github/workflows/encryption-tests.yml

@@ -0,0 +1,39 @@
+name: Encryption Tests 
+
+on:
+  push: 
+    branches: ['master']
+  pull_request:
+    branches: [ 'master' ]
+  workflow_dispatch: {}
+
+permissions:
+  contents: write
+  pull-requests: write
+  id-token: write
+
+jobs:
+  run-tests:
+    permissions:
+      # required for all workflows
+      security-events: write
+      id-token: write
+      contents: write
+    runs-on: ubuntu-latest
+    name: Encryption tests
+    env:
+      FORCE_COLOR: true
+    steps:
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+      - name: Setup node
+        uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4
+        with:
+          node-version: 22
+      - name: Install Dependencies
+        run: npm install
+      - name: Install mongodb-client-encryption
+        run: npm install mongodb-client-encryption
+      - name: Setup Tests
+        run: npm run setup-test-encryption
+      - name: Run Tests
+        run: npm run test-encryption

.gitignore

@@ -67,3 +67,6 @@ examples/ecommerce-netlify-functions/.netlify/state.json
 
 notes.md
 list.out
+
+data
+*.pid

CONTRIBUTING.md

@@ -46,6 +46,13 @@ If you have a question about Mongoose (not a bug report) please post it to eithe
   * execute `npm run test-tsd` to run the typescript tests
   * execute `npm run ts-benchmark` to run the typescript benchmark "performance test" for a single time.
   * execute `npm run ts-benchmark-watch` to run the typescript benchmark "performance test" while watching changes on types folder. Note: Make sure to commit all changes before executing this command.
+* in order to run tests that require an cluster with encryption locally, run `npm run setup-test-encryption` followed by `npm run test-encryption`. Alternatively, you can start an encrypted cluster using the `scripts/configure-cluster-with-encryption.sh` file.
+  * These scripts can take a few minutes to run.
+  * To change an encryption configuration, it is recommended to follow these steps:
+    * Edit the variables in `scripts/configure-cluster-with-encryption.sh` with your desired configuration.
+    * Restart your shell.
+    * Delete the `data/` directory if it exists.
+    * Finally, run the configuration script.
 
 ## Documentation
 

package.json

@@ -104,6 +104,8 @@
     "test-deno": "deno run --allow-env --allow-read --allow-net --allow-run --allow-sys --allow-write ./test/deno.js",
     "test-rs": "START_REPLICA_SET=1 mocha --timeout 30000 --exit ./test/*.test.js",
     "test-tsd": "node ./test/types/check-types-filename && tsd",
+    "setup-test-encryption": "bash scripts/configure-cluster-with-encryption.sh",
+    "test-encryption": "mocha --exit ./test/encryption/*.test.js",
     "tdd": "mocha ./test/*.test.js --inspect --watch --recursive --watch-files ./**/*.{js,ts}",
     "test-coverage": "nyc --reporter=html --reporter=text npm test",
     "ts-benchmark": "cd ./benchmarks/typescript/simple && npm install && npm run benchmark | node ../../../scripts/tsc-diagnostics-check"

scripts/configure-cluster-with-encryption.sh

@@ -0,0 +1,54 @@
+# note: in order to use FLE with mongodb, we must
+#  have mongocryptd or the shared library downloaded
+#  have an enterprise server >= 4.2
+
+# this script downloads all tools required to use FLE with mongodb, then starts a cluster of the provided configuration (sharded on 8.0 server)
+
+export CWD=$(pwd);
+
+# install extra dependency 
+npm install mongodb-client-encryption
+
+# set up mongodb cluster and encryption configuration if the data/ folder does not exist
+if [ ! -d "data" ]; then
+
+  mkdir data
+  cd data
+
+  # note: 
+    # we're using drivers-evergreen-tools which is a repo used by MongoDB drivers to start clusters for testing.  
+    # if you'd like to make changes to the cluster settings, edit the exported variables below.
+    # for configuration options for the exported variables, see here: https://github.com/mongodb-labs/drivers-evergreen-tools/blob/master/.evergreen/run-orchestration.sh
+    # after this script is run, the data/ folder will notably contain the following:
+      # 'mo-expansion.yml' file which contains for your cluster URI and crypt shared library path
+      # 'drivers-evergreen-tools/mongodb/bin' which contain executables for other mongodb libraries such as mongocryptd, mongosh, and mongod
+  if [ ! -d "drivers-evergreen-tools/" ]; then
+  git clone --depth=1 "https://github.com/mongodb-labs/drivers-evergreen-tools.git"
+  fi
+
+  # configure cluster settings
+  export DRIVERS_TOOLS=$CWD/data/drivers-evergreen-tools
+  export MONGODB_VERSION=8.0
+  export AUTH=true
+  export MONGODB_BINARIES=$DRIVERS_TOOLS/mongodb/bin
+  export MONGO_ORCHESTRATION_HOME=$DRIVERS_TOOLS/mo
+  export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration
+  export TOPOLOGY=sharded_cluster
+  export SSL=nossl
+
+  cd $DRIVERS_TOOLS
+  rm -rf mongosh mongodb mo
+  mkdir mo
+  cd -
+
+  rm expansions.sh 2> /dev/null
+
+  echo 'Configuring Cluster...'
+
+  # start cluster
+  (bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh) 1> /dev/null 2> /dev/null
+ 
+  echo 'Cluster Configuration Finished!'
+
+  cd ..
+fi

test/encryption/encryption.test.js

@@ -0,0 +1,115 @@
+'use strict';
+
+const assert = require('assert');
+const mongodb = require('mongodb');
+const fs = require('fs');
+const isBsonType = require('../../lib/helpers/isBsonType');
+
+const LOCAL_KEY = Buffer.from('Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', 'base64');
+
+describe('ci', () => {
+
+  const cachedUri = process.env.MONGOOSE_TEST_URI;
+  const cachedLib = process.env.CRYPT_SHARED_LIB_PATH;
+
+  before(function() {
+    const cwd = process.cwd();
+    const file = fs.readFileSync(cwd + '/data/mo-expansion.yml', { encoding: 'utf-8' }).trim().split('\n');
+    const regex = /^(?<key>.*): "(?<value>.*)"$/;
+    const variables = file.map((line) => regex.exec(line.trim()).groups).reduce((acc, { key, value }) => ({ ...acc, [key]: value }), {});
+    process.env.CRYPT_SHARED_LIB_PATH = variables.CRYPT_SHARED_LIB_PATH;
+    process.env.MONGOOSE_TEST_URI = variables.MONGODB_URI;
+  });
+
+  after(function() {
+    process.env.CRYPT_SHARED_LIB_PATH = cachedLib;
+    process.env.MONGOOSE_TEST_URI = cachedUri;
+  });
+
+  describe('environmental variables', () => {
+    it('MONGOOSE_TEST_URI is set', async function() {
+      const uri = process.env.MONGOOSE_TEST_URI;
+      assert.ok(uri);
+    });
+
+    it('CRYPT_SHARED_LIB_PATH is set', async function() {
+      const shared_library_path = process.env.CRYPT_SHARED_LIB_PATH;
+      assert.ok(shared_library_path);
+    });
+  });
+
+  describe('basic integration', () => {
+    let keyVaultClient;
+    let dataKey;
+    let encryptedClient;
+    let unencryptedClient;
+
+    beforeEach(async function() {
+      keyVaultClient = new mongodb.MongoClient(process.env.MONGOOSE_TEST_URI);
+      await keyVaultClient.connect();
+      await keyVaultClient.db('keyvault').collection('datakeys');
+      const clientEncryption = new mongodb.ClientEncryption(keyVaultClient, {
+        keyVaultNamespace: 'keyvault.datakeys',
+        kmsProviders: { local: { key: LOCAL_KEY } }
+      });
+      dataKey = await clientEncryption.createDataKey('local');
+
+      encryptedClient = new mongodb.MongoClient(
+        process.env.MONGOOSE_TEST_URI,
+        {
+          autoEncryption: {
+            keyVaultNamespace: 'keyvault.datakeys',
+            kmsProviders: { local: { key: LOCAL_KEY } },
+            schemaMap: {
+              'db.coll': {
+                bsonType: 'object',
+                encryptMetadata: {
+                  keyId: [dataKey]
+                },
+                properties: {
+                  a: {
+                    encrypt: {
+                      bsonType: 'int',
+                      algorithm: 'AEAD_AES_256_CBC_HMAC_SHA_512-Random',
+                      keyId: [dataKey]
+                    }
+                  }
+                }
+              }
+            },
+            extraOptions: {
+              cryptdSharedLibRequired: true,
+              cryptSharedLibPath: process.env.CRYPT_SHARED_LIB_PATH
+            }
+          }
+        }
+      );
+
+      unencryptedClient = new mongodb.MongoClient(process.env.MONGOOSE_TEST_URI);
+    });
+
+    afterEach(async function() {
+      await keyVaultClient.close();
+      await encryptedClient.close();
+      await unencryptedClient.close();
+    });
+
+    it('ci set-up should support basic mongodb auto-encryption integration', async() => {
+      await encryptedClient.connect();
+      const { insertedId } = await encryptedClient.db('db').collection('coll').insertOne({ a: 1 });
+
+      // client not configured with autoEncryption, returns a encrypted binary type, meaning that encryption succeeded
+      const encryptedResult = await unencryptedClient.db('db').collection('coll').findOne({ _id: insertedId });
+
+      assert.ok(encryptedResult);
+      assert.ok(encryptedResult.a);
+      assert.ok(isBsonType(encryptedResult.a, 'Binary'));
+      assert.ok(encryptedResult.a.sub_type === 6);
+
+      // when the encryptedClient runs a find, the original unencrypted value is returned
+      const unencryptedResult = await encryptedClient.db('db').collection('coll').findOne({ _id: insertedId });
+      assert.ok(unencryptedResult);
+      assert.ok(unencryptedResult.a === 1);
+    });
+  });
+});