support job log level in lightning plan

OpenFn · Feb 11, 2025 · 097d58e · 097d58e
1 parent 9202fb2
commit 097d58e
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 7 deletions.
diff --git a/packages/engine-multi/src/worker/thread/helpers.ts b/packages/engine-multi/src/worker/thread/helpers.ts
@@ -3,7 +3,7 @@
 import process from 'node:process';
 import stringify from 'fast-safe-stringify';
 import createLogger, { SanitizePolicies } from '@openfn/logger';
-import type { JSONLog } from '@openfn/logger';
+import type { JSONLog, LogLevel } from '@openfn/logger';
 
 import * as workerEvents from '../events';
 import { HANDLED_EXIT_CODE } from '../../events';
@@ -14,7 +14,8 @@ import serializeError from '../../util/serialize-error';
 export const createLoggers = (
   workflowId: string,
   sanitize: SanitizePolicies = 'none',
-  publish?: any
+  publish?: any,
+  jobLogLevel: LogLevel = 'debug'
 ) => {
   const log = (message: JSONLog) => {
     publish(workerEvents.LOG, {
@@ -47,7 +48,7 @@ export const createLoggers = (
 
   const jobLogger = createLogger('JOB', {
     logger: emitter,
-    level: 'debug',
+    level: jobLogLevel,
     json: true,
     sanitize,
   });

diff --git a/packages/engine-multi/src/worker/thread/run.ts b/packages/engine-multi/src/worker/thread/run.ts
@@ -3,7 +3,7 @@
 import run from '@openfn/runtime';
 import type { NotifyEvents } from '@openfn/runtime';
 import type { ExecutionPlan, State } from '@openfn/lexicon';
-import type { SanitizePolicies } from '@openfn/logger';
+import type { LogLevel, SanitizePolicies } from '@openfn/logger';
 
 import { register, publish } from './runtime';
 import { execute, createLoggers } from './helpers';
@@ -15,6 +15,7 @@ export type RunOptions = {
   whitelist?: RegExp[];
   sanitize: SanitizePolicies;
   statePropsToRemove?: string[];
+  jobLogLevel?: LogLevel;
 };
 
 const eventMap = {
@@ -26,11 +27,13 @@ const eventMap = {
 
 register({
   run: (plan: ExecutionPlan, input: State, runOptions: RunOptions) => {
-    const { repoDir, whitelist, sanitize, statePropsToRemove } = runOptions;
+    const { repoDir, whitelist, sanitize, statePropsToRemove, jobLogLevel } =
+      runOptions;
     const { logger, jobLogger, adaptorLogger } = createLoggers(
       plan.id!,
       sanitize,
-      publish
+      publish,
+      jobLogLevel
     );
 
     // Save the debug function so that we can use it

diff --git a/packages/lexicon/lightning.d.ts b/packages/lexicon/lightning.d.ts
@@ -1,4 +1,4 @@
-import type { SanitizePolicies } from '@openfn/logger';
+import type { LogLevel, SanitizePolicies } from '@openfn/logger';
 import { LegacyJob, State } from './core';
 
 export const API_VERSION: number;
@@ -50,6 +50,7 @@ export type LightningPlanOptions = {
 
   run_memory_limit_mb?: number;
   payload_limit_mb?: number;
+  job_log_level?: LogLevel;
 };
 
 /**

diff --git a/packages/ws-worker/src/util/convert-lightning-plan.ts b/packages/ws-worker/src/util/convert-lightning-plan.ts
@@ -11,6 +11,7 @@ import type {
   WorkflowOptions,
   Lazy,
 } from '@openfn/lexicon';
+import type { LogLevel } from '@openfn/logger';
 import { LightningPlan, LightningEdge } from '@openfn/lexicon/lightning';
 import { ExecuteOptions } from '@openfn/engine-multi';
 import { getNameAndVersion } from '@openfn/runtime';
@@ -46,6 +47,7 @@ export type WorkerRunOptions = ExecuteOptions & {
   // Defaults to true - must be explicity false to stop dataclips being sent
   outputDataclips?: boolean;
   payloadLimitMb?: number;
+  jobLogLevel?: LogLevel;
 };
 
 type ConversionOptions = {
@@ -124,6 +126,9 @@ export default (
     if ('output_dataclips' in run.options) {
       engineOpts.outputDataclips = run.options.output_dataclips;
     }
+    if ('job_log_level' in run.options) {
+      engineOpts.jobLogLevel = run.options.job_log_level;
+    }
   }
 
   const plan: Partial<ExecutionPlan> = {