[FAI-16250] Allow users to pass in additional docker volume mount (#135)

Author: jeniii

airbyte-local-cli-nodejs/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@faros-ai/airbyte-local-cli-nodejs",
-  "version": "0.0.5",
+  "version": "0.0.6",
   "description": "Airbyte local cli node js version",
   "private": true,
   "packageManager": "^[email protected]",

airbyte-local-cli-nodejs/package.json

@@ -14,6 +14,8 @@ import {
   TMP_SPEC_CONFIG_FILENAME,
   TMP_WIZARD_CONFIG_FILENAME,
 } from './constants/constants';
+// Self-reference for unit test mocking
+import * as docker from './docker';
 import {logger} from './logger';
 import {
   AirbyteCatalog,
@@ -34,8 +36,8 @@ const DEFAULT_MAX_LOG_SIZE = '10m';
 let _docker = new Docker();
 
 // For testing purposes
-export function setDocker(docker: Docker): void {
-  _docker = docker;
+export function setDocker(testDocker: Docker): void {
+  _docker = testDocker;
 }
 
 /**
@@ -182,7 +184,7 @@ export function processSpecByLine(line: string): AirbyteSpec | undefined {
  * @param options - Docker container create options
  * @param outputStream - Writable stream to capture the output
  */
-async function runDocker(
+export async function runDocker(
   options: Docker.ContainerCreateOptions,
   outputStream: Writable,
   inputStream?: ReadStream | PassThrough,
@@ -288,7 +290,7 @@ export async function runCheckSrcConnection(tmpDir: string, image: string, srcCo
     });
 
     // run docker
-    await runDocker(createOptions, containerOutputStream);
+    await docker.runDocker(createOptions, containerOutputStream);
 
     // capture connection status from the output
     let status: AirbyteConnectionStatusMessage | undefined;
@@ -348,7 +350,7 @@ export async function runDiscoverCatalog(tmpDir: string, image: string | undefin
     });
 
     // run docker
-    await runDocker(createOptions, containerOutputStream);
+    await docker.runDocker(createOptions, containerOutputStream);
 
     // capture catalog output
     let rawCatalog: AirbyteCatalogMessage | undefined;
@@ -438,8 +440,14 @@ export async function runSrcSync(tmpDir: string, config: FarosConfig, srcOutputS
           },
         },
         ...config.src?.dockerOptions?.additionalOptions?.HostConfig,
+
+        // Allow users to add bind mount but not override the default one
+        Binds: [
+          `${tmpDir}:${getBindsLocation(config.src.image)}`,
+          ...(config.src?.dockerOptions?.additionalOptions?.HostConfig?.Binds || []),
+        ],
+
         // Default options: cannot be overridden by users
-        Binds: [`${tmpDir}:${getBindsLocation(config.src.image)}`],
         AutoRemove: !config.keepContainers,
         Init: true,
       },
@@ -467,7 +475,7 @@ export async function runSrcSync(tmpDir: string, config: FarosConfig, srcOutputS
     });
 
     // run docker
-    await runDocker(createOptions, containerOutputStream);
+    await docker.runDocker(createOptions, containerOutputStream);
 
     // Close the container output stream
     // This is required to notify the dst connector that inputs are done
@@ -564,8 +572,14 @@ export async function runDstSync(tmpDir: string, config: FarosConfig, srcPassThr
           },
         },
         ...config.dst?.dockerOptions?.additionalOptions?.HostConfig,
+
+        // Allow users to add bind mount but not override the default one
+        Binds: [
+          `${tmpDir}:${getBindsLocation(config.dst.image)}`,
+          ...(config.dst?.dockerOptions?.additionalOptions?.HostConfig?.Binds || []),
+        ],
+
         // Default options: cannot be overridden by users
-        Binds: [`${tmpDir}:${getBindsLocation(config.dst.image)}`],
         AutoRemove: !config.keepContainers,
         Init: true,
       },
@@ -593,7 +607,7 @@ export async function runDstSync(tmpDir: string, config: FarosConfig, srcPassThr
     const inputStream = srcPassThrough ?? createReadStream(`${tmpDir}/${SRC_OUTPUT_DATA_FILE}`);
 
     // Start the container
-    await runDocker(createOptions, containerOutputStream, inputStream);
+    await docker.runDocker(createOptions, containerOutputStream, inputStream);
     logger.info('Destination connector completed.');
 
     // Write the state file
@@ -651,7 +665,7 @@ export async function runSpec(image: string): Promise<AirbyteSpec> {
     });
 
     // run docker
-    await runDocker(createOptions, containerOutputStream);
+    await docker.runDocker(createOptions, containerOutputStream);
 
     // write spec to the file
     if (specs.length > 0) {
@@ -713,7 +727,7 @@ export async function runWizard(tmpDir: string, image: string, spec: AirbyteSpec
     };
 
     // run docker
-    await runDocker(createOptions, process.stdout);
+    await docker.runDocker(createOptions, process.stdout);
 
     const resultConfig = JSON.parse(readFileSync(`${tmpDir}/${TMP_WIZARD_CONFIG_FILENAME}`, 'utf-8'));
     return resultConfig;

airbyte-local-cli-nodejs/src/docker.ts

@@ -1 +1 @@
-export const CLI_VERSION = '0.0.5';
+export const CLI_VERSION = '0.0.6';

airbyte-local-cli-nodejs/src/version.ts

@@ -93,6 +93,13 @@ exports[`parseConfigFile should parse utf16 encoding 1`] = `
       "username": "test-username",
     },
     "dockerOptions": {
+      "additionalOptions": {
+        "HostConfig": {
+          "Binds": [
+            "/test/path:/test/path/test.json",
+          ],
+        },
+      },
       "maxCpus": 2,
       "maxMemory": 2048,
     },
@@ -127,6 +134,13 @@ exports[`parseConfigFile should pass 1`] = `
       "username": "test-username",
     },
     "dockerOptions": {
+      "additionalOptions": {
+        "HostConfig": {
+          "Binds": [
+            "/test/path:/test/path/test.json",
+          ],
+        },
+      },
       "maxCpus": 2,
       "maxMemory": 2048,
     },

airbyte-local-cli-nodejs/test/__snapshots__/utils.it.test.ts.snap

@@ -0,0 +1,55 @@
+import * as docker from '../src/docker';
+import {FarosConfig} from '../src/types';
+
+describe('runSrcSync', () => {
+  const testCfg: FarosConfig = {
+    src: {
+      image: 'farosai/airbyte-example-source',
+    },
+    srcCheckConnection: false,
+    dstUseHostNetwork: false,
+    srcPull: false,
+    dstPull: false,
+    fullRefresh: false,
+    rawMessages: false,
+    keepContainers: false,
+    logLevel: 'info',
+    debug: false,
+    stateFile: undefined,
+    connectionName: undefined,
+    srcOutputFile: undefined,
+    srcInputFile: undefined,
+    silent: false,
+    image: false,
+  };
+
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it('should call runDocker with correct parameters', async () => {
+    const mockRunDocker = jest.spyOn(docker, 'runDocker').mockResolvedValue(undefined);
+
+    const testCfgWithVolumeMount = {
+      ...testCfg,
+      src: {
+        ...testCfg.src,
+        dockerOptions: {
+          additionalOptions: {
+            HostConfig: {
+              Binds: ['/path/to/tasks.xlsx:/tasks.xlsx'],
+            },
+          },
+        },
+      },
+    } as FarosConfig;
+
+    await docker.runSrcSync('testTmpDir', testCfgWithVolumeMount, process.stdout);
+
+    expect(mockRunDocker).toHaveBeenCalled();
+    expect(mockRunDocker.mock.calls[0]?.[0]?.HostConfig?.Binds).toMatchObject([
+      'testTmpDir:/configs',
+      '/path/to/tasks.xlsx:/tasks.xlsx',
+    ]);
+  });
+});

airbyte-local-cli-nodejs/test/docker.test.ts

@@ -9,7 +9,14 @@
     "catalog": {},
     "dockerOptions": {
       "maxMemory": 2048,
-      "maxCpus": 2
+      "maxCpus": 2,
+      "additionalOptions": {
+        "HostConfig": {
+          "Binds": [
+            "/test/path:/test/path/test.json"
+          ]
+        }
+      }
     }
   },
   "dst": {