Release/next (#881)

* fix: wrong formatting of run complete time (#878) * fix: wrong formatting of run complete time logger.timer already returns a better human readable time string. * changeset --------- Co-authored-by: Joe Clark <[email protected]> * Fix worker log level (#880) * ensure the jobLogLevel option gets properly fed through to the worker * fix integration test and tidy logging * version: [email protected] [email protected] * worker: remove debug code --------- Co-authored-by: Farhan Y. <[email protected]>
OpenFn · Feb 25, 2025 · b4f0f12 · b4f0f12
1 parent 1a0a712
commit b4f0f12
Show file tree

Hide file tree

Showing 21 changed files with 113 additions and 25 deletions.
diff --git a/integration-tests/execute/CHANGELOG.md b/integration-tests/execute/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/integration-tests-execute
 
+## 1.0.17
+
+### Patch Changes
+
+- Updated dependencies [2667710]
+  - @openfn/runtime@1.6.3
+
 ## 1.0.16
 
 ### Patch Changes

diff --git a/integration-tests/execute/package.json b/integration-tests/execute/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@openfn/integration-tests-execute",
   "private": true,
-  "version": "1.0.16",
+  "version": "1.0.17",
   "description": "Job execution tests",
   "author": "Open Function Group <[email protected]>",
   "license": "ISC",

diff --git a/integration-tests/worker/CHANGELOG.md b/integration-tests/worker/CHANGELOG.md
@@ -1,5 +1,15 @@
 # @openfn/integration-tests-worker
 
+## 1.0.78
+
+### Patch Changes
+
+- Updated dependencies [d430258]
+- Updated dependencies [2667710]
+  - @openfn/engine-multi@1.5.1
+  - @openfn/ws-worker@1.11.1
+  - @openfn/lightning-mock@2.1.1
+
 ## 1.0.77
 
 ### Patch Changes

diff --git a/integration-tests/worker/package.json b/integration-tests/worker/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@openfn/integration-tests-worker",
   "private": true,
-  "version": "1.0.77",
+  "version": "1.0.78",
   "description": "Lightning WOrker integration tests",
   "author": "Open Function Group <[email protected]>",
   "license": "ISC",

diff --git a/integration-tests/worker/src/init.ts b/integration-tests/worker/src/init.ts
@@ -5,7 +5,7 @@ import createLightningServer, { toBase64 } from '@openfn/lightning-mock';
 import createEngine from '@openfn/engine-multi';
 import createWorkerServer from '@openfn/ws-worker';
 import { createMockLogger } from '@openfn/logger';
-import createLogger from '@openfn/logger';
+// import createLogger from '@openfn/logger';
 
 export const randomPort = () => Math.round(2000 + Math.random() * 1000);
 
@@ -39,8 +39,8 @@ export const initWorker = async (
   });
 
   const worker = createWorkerServer(engine, {
-    // logger: createMockLogger(),
-    logger: createLogger('worker', { level: 'debug' }),
+    logger: createMockLogger(),
+    // logger: createLogger('worker', { level: 'debug' }),
     port: workerPort,
     lightning: `ws://localhost:${lightningPort}/worker`,
     secret: crypto.randomUUID(),

diff --git a/integration-tests/worker/test/integration.test.ts b/integration-tests/worker/test/integration.test.ts
@@ -996,19 +996,23 @@ test.serial(
         },
       };
 
+      const jobLogs = [];
+      const rtLogs = [];
+
+      lightning.on('run:log', (e) => {
+        if (e.payload.source === 'JOB') {
+          jobLogs.push(e.payload);
+        } else if (e.payload.source === 'R/T') {
+          rtLogs.push(e.payload);
+        }
+      });
+
       lightning.once('run:complete', () => {
-        const jsonLogs = engineLogger._history;
-        // The engine logger shouldn't print out any job logs
-        const jobLog = jsonLogs.find((l) => l.name === 'JOB');
-        t.falsy(jobLog);
-        const jobLog2 = jsonLogs.find((l) => l.message[0] === message);
-        t.falsy(jobLog2);
+        // Ensure no run logs got sent back to lightning
+        t.falsy(jobLogs.length);
 
         // But it SHOULD log engine stuff
-        const runtimeLog = jsonLogs.find(
-          (l) => l.name === 'engine' && l.message[0].match(/complete workflow/i)
-        );
-        t.truthy(runtimeLog);
+        t.truthy(rtLogs.length);
         done();
       });
 

diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md
@@ -1,5 +1,13 @@
 # @openfn/cli
 
+## 1.11.3
+
+### Patch Changes
+
+- 2667710: Fix an issue where step completion time is logged with double units (ie, `2msms`)
+- Updated dependencies [2667710]
+  - @openfn/runtime@1.6.3
+
 ## 1.11.2
 
 ### Patch Changes

diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.11.2",
+  "version": "1.11.3",
   "description": "CLI devtools for the openfn toolchain.",
   "engines": {
     "node": ">=18",

diff --git a/packages/engine-multi/CHANGELOG.md b/packages/engine-multi/CHANGELOG.md
@@ -1,5 +1,13 @@
 # engine-multi
 
+## 1.5.1
+
+### Patch Changes
+
+- d430258: Fix an issue where Lightning log level options don't get fed to the engine properly
+- Updated dependencies [2667710]
+  - @openfn/runtime@1.6.3
+
 ## 1.5.0
 
 ### Minor Changes

diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/engine-multi",
-  "version": "1.5.0",
+  "version": "1.5.1",
   "description": "Multi-process runtime engine",
   "main": "dist/index.js",
   "type": "module",

diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts
@@ -50,6 +50,7 @@ const execute = async (context: ExecutionContext) => {
     const runOptions = {
       statePropsToRemove: options.statePropsToRemove,
       whitelist,
+      jobLogLevel: options.jobLogLevel,
     } as RunOptions;
 
     const workerOptions = {
@@ -108,7 +109,6 @@ const execute = async (context: ExecutionContext) => {
         jobError(context, evt);
       },
       [workerEvents.LOG]: (evt: workerEvents.LogEvent) => {
-        // console.log(evt.log.name, evt.log.message);
         log(context, evt);
       },
       // TODO this is also untested

diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts
@@ -173,6 +173,7 @@ const createEngine = async (
         resolvers: opts.resolvers,
         runTimeoutMs: opts.runTimeoutMs ?? defaultTimeout,
         memoryLimitMb: opts.memoryLimitMb ?? defaultMemoryLimit,
+        jobLogLevel: opts.jobLogLevel,
       },
     });
 

diff --git a/packages/engine-multi/src/types.ts b/packages/engine-multi/src/types.ts
@@ -49,11 +49,10 @@ export type ExecuteOptions = {
   resolvers?: LazyResolvers;
   runTimeoutMs?: number;
   sanitize?: SanitizePolicies;
+  jobLogLevel?: string;
 };
 
-export type ExecutionContextOptions = EngineOptions & {
-  sanitize?: SanitizePolicies;
-};
+export type ExecutionContextOptions = ExecuteOptions & EngineOptions;
 
 export interface EngineAPI extends EventEmitter {
   callWorker: CallWorker;

diff --git a/packages/engine-multi/test/api/execute.test.ts b/packages/engine-multi/test/api/execute.test.ts
@@ -347,3 +347,28 @@ test.serial('should stringify the whitelist array', async (t) => {
   t.truthy(passedOptions);
   t.deepEqual(passedOptions.whitelist, ['/abc/']);
 });
+
+test.serial('should forward the jobLogLevel option', async (t) => {
+  let passedOptions: any;
+
+  const state = {
+    id: 'x',
+    plan,
+  } as WorkflowState;
+
+  const opts = {
+    ...options,
+    jobLogLevel: 'none',
+  };
+
+  const context = createContext({ state, options: opts });
+  // @ts-ignore
+  context.callWorker = (_command, args) => {
+    passedOptions = args[2];
+  };
+
+  await execute(context);
+
+  t.truthy(passedOptions);
+  t.deepEqual(passedOptions.jobLogLevel, 'none');
+});
diff --git a/packages/lightning-mock/CHANGELOG.md b/packages/lightning-mock/CHANGELOG.md
@@ -1,5 +1,14 @@
 # @openfn/lightning-mock
 
+## 2.1.1
+
+### Patch Changes
+
+- Updated dependencies [d430258]
+- Updated dependencies [2667710]
+  - @openfn/engine-multi@1.5.1
+  - @openfn/runtime@1.6.3
+
 ## 2.1.0
 
 ### Minor Changes

diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/lightning-mock",
-  "version": "2.1.0",
+  "version": "2.1.1",
   "private": true,
   "description": "A mock Lightning server",
   "main": "dist/index.js",

diff --git a/packages/runtime/CHANGELOG.md b/packages/runtime/CHANGELOG.md
@@ -1,5 +1,11 @@
 # @openfn/runtime
 
+## 1.6.3
+
+### Patch Changes
+
+- 2667710: Fix an issue where step completion time is logged with double units (ie, `2msms`)
+
 ## 1.6.2
 
 ### Patch Changes

diff --git a/packages/runtime/package.json b/packages/runtime/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/runtime",
-  "version": "1.6.2",
+  "version": "1.6.3",
   "description": "Job processing runtime.",
   "type": "module",
   "exports": {

diff --git a/packages/runtime/src/execute/step.ts b/packages/runtime/src/execute/step.ts
@@ -207,7 +207,7 @@ const executeStep = async (
 
     if (!didError) {
       const humanDuration = logger.timer(timerId);
-      logger.success(`${jobName} completed in ${humanDuration}ms`);
+      logger.success(`${jobName} completed in ${humanDuration}`);
       result = prepareFinalState(result, logger, ctx.opts.statePropsToRemove);
 
       // Take a memory snapshot

diff --git a/packages/ws-worker/CHANGELOG.md b/packages/ws-worker/CHANGELOG.md
@@ -1,5 +1,16 @@
 # ws-worker
 
+## 1.11.1
+
+### Patch Changes
+
+- d430258: Fix an issue where Lightning log level options don't get fed to the engine properly
+- 2667710: Fix an issue where step completion time is logged with double units (ie, `2msms`)
+- Updated dependencies [d430258]
+- Updated dependencies [2667710]
+  - @openfn/engine-multi@1.5.1
+  - @openfn/runtime@1.6.3
+
 ## 1.11.0
 
 ### Minor Changes

diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "1.11.0",
+  "version": "1.11.1",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",