diff --git a/dashboard/apps/web/app/wfspec/[id]/[version]/components/visualizer/mappers/GraphLayouter.test.ts b/dashboard/apps/web/app/wfspec/[id]/[version]/components/visualizer/mappers/GraphLayouter.test.ts index c520a095b..1c76948db 100644 --- a/dashboard/apps/web/app/wfspec/[id]/[version]/components/visualizer/mappers/GraphLayouter.test.ts +++ b/dashboard/apps/web/app/wfspec/[id]/[version]/components/visualizer/mappers/GraphLayouter.test.ts @@ -7,7 +7,6 @@ import { LHStatus, MetadataStatus, VariableType, - WaitForThreadsPolicy } from '../../../../../../../littlehorse-public-api/common_enums' import type { TaskNode, @@ -2795,14 +2794,16 @@ describe('Layouting graph from LH Nodes', () => { ], 'failureHandlers': [], 'waitForThreads': { - 'threads': [ - { - 'threadRunNumber': { - 'variableName': '2-spawned-thread-START_THREAD' + 'threads': { + threads: [ + { + 'threadRunNumber': { + 'variableName': '2-spawned-thread-START_THREAD' + } } - } - ], - 'policy': WaitForThreadsPolicy.STOP_ON_FAILURE + ] + }, + perThreadFailureHandlers: [] } }, '4-parent-task-2-TASK': { diff --git a/dashboard/apps/web/components/Drawer/internals/ThreadRunsHandler.test.ts b/dashboard/apps/web/components/Drawer/internals/ThreadRunsHandler.test.ts index 599c40dbe..8fe7259ec 100644 --- a/dashboard/apps/web/components/Drawer/internals/ThreadRunsHandler.test.ts +++ b/dashboard/apps/web/components/Drawer/internals/ThreadRunsHandler.test.ts @@ -1,8 +1,8 @@ import { faker } from '@faker-js/faker' -import type { NodeRun } from '../../../littlehorse-public-api/node_run' +import { WaitForThreadsRun_WaitingThreadStatus, type NodeRun } from '../../../littlehorse-public-api/node_run' import type { ThreadRun } from '../../../littlehorse-public-api/wf_run' import { ThreadType } from '../../../littlehorse-public-api/wf_run' -import { LHStatus, WaitForThreadsPolicy } from '../../../littlehorse-public-api/common_enums' +import { LHStatus } from '../../../littlehorse-public-api/common_enums' import ThreadRunsHandler from './ThreadRunsHandler' describe('handle Thread Runs related logic', () => { @@ -86,15 +86,14 @@ describe('handle Thread Runs related logic', () => { { 'threadStatus': LHStatus.RUNNING, 'threadRunNumber': 1, - 'alreadyHandled': false + 'waitingStatus': WaitForThreadsRun_WaitingThreadStatus.THREAD_IN_PROGRESS }, { 'threadStatus': LHStatus.RUNNING, 'threadRunNumber': 2, - 'alreadyHandled': false + 'waitingStatus': WaitForThreadsRun_WaitingThreadStatus.THREAD_IN_PROGRESS } - ], - 'policy': WaitForThreadsPolicy.STOP_ON_FAILURE + ] }, 'failureHandlerIds': [] } @@ -150,10 +149,9 @@ describe('handle Thread Runs related logic', () => { { 'threadStatus': LHStatus.RUNNING, 'threadRunNumber': 1, - 'alreadyHandled': false + 'waitingStatus': WaitForThreadsRun_WaitingThreadStatus.THREAD_IN_PROGRESS } - ], - 'policy': WaitForThreadsPolicy.STOP_ON_FAILURE + ] }, 'failureHandlerIds': [] } diff --git a/dashboard/apps/web/littlehorse-public-api/common_enums.ts b/dashboard/apps/web/littlehorse-public-api/common_enums.ts index f0f959c89..dfb636d22 100644 --- a/dashboard/apps/web/littlehorse-public-api/common_enums.ts +++ b/dashboard/apps/web/littlehorse-public-api/common_enums.ts @@ -516,40 +516,3 @@ export function lHErrorTypeToNumber(object: LHErrorType): number { return -1; } } - -export enum WaitForThreadsPolicy { - STOP_ON_FAILURE = "STOP_ON_FAILURE", - UNRECOGNIZED = "UNRECOGNIZED", -} - -export function waitForThreadsPolicyFromJSON(object: any): WaitForThreadsPolicy { - switch (object) { - case 0: - case "STOP_ON_FAILURE": - return WaitForThreadsPolicy.STOP_ON_FAILURE; - case -1: - case "UNRECOGNIZED": - default: - return WaitForThreadsPolicy.UNRECOGNIZED; - } -} - -export function waitForThreadsPolicyToJSON(object: WaitForThreadsPolicy): string { - switch (object) { - case WaitForThreadsPolicy.STOP_ON_FAILURE: - return "STOP_ON_FAILURE"; - case WaitForThreadsPolicy.UNRECOGNIZED: - default: - return "UNRECOGNIZED"; - } -} - -export function waitForThreadsPolicyToNumber(object: WaitForThreadsPolicy): number { - switch (object) { - case WaitForThreadsPolicy.STOP_ON_FAILURE: - return 0; - case WaitForThreadsPolicy.UNRECOGNIZED: - default: - return -1; - } -} diff --git a/dashboard/apps/web/littlehorse-public-api/node_run.ts b/dashboard/apps/web/littlehorse-public-api/node_run.ts index 6754ccd2f..6bb8027f1 100644 --- a/dashboard/apps/web/littlehorse-public-api/node_run.ts +++ b/dashboard/apps/web/littlehorse-public-api/node_run.ts @@ -1,15 +1,6 @@ /* eslint-disable */ import * as _m0 from "protobufjs/minimal"; -import { - LHStatus, - lHStatusFromJSON, - lHStatusToJSON, - lHStatusToNumber, - WaitForThreadsPolicy, - waitForThreadsPolicyFromJSON, - waitForThreadsPolicyToJSON, - waitForThreadsPolicyToNumber, -} from "./common_enums"; +import { LHStatus, lHStatusFromJSON, lHStatusToJSON, lHStatusToNumber } from "./common_enums"; import { Timestamp } from "./google/protobuf/timestamp"; import { ExternalEventDefId, ExternalEventId, NodeRunId, TaskRunId, UserTaskRunId, WfSpecId } from "./object_id"; import { VariableValue } from "./variable"; @@ -157,11 +148,82 @@ export interface StartMultipleThreadsRun { export interface WaitForThreadsRun { /** The threads that are being waited for. */ threads: WaitForThreadsRun_WaitForThread[]; +} + +/** The status of a single ThreadRun that we are waiting for. */ +export enum WaitForThreadsRun_WaitingThreadStatus { + /** THREAD_IN_PROGRESS - The ThreadRun is in progress (i.e. not COMPLETED nor EXCEPTION nor ERROR) */ + THREAD_IN_PROGRESS = "THREAD_IN_PROGRESS", /** - * The policy to use when handling failures for Threads. Currently, only - * one policy exists. + * THREAD_HANDLING_FAILURE - The ThreadRun failed with some failure, and the FailureHandler is running + * for that Failure. */ - policy: WaitForThreadsPolicy; + THREAD_HANDLING_FAILURE = "THREAD_HANDLING_FAILURE", + /** + * THREAD_COMPLETED_OR_FAILURE_HANDLED - We can mark this ThreadRun as "already waited for", meaning that either: + * 1. It completed successfully, OR + * 2. It failed, and the Failure Handler successfully completed + */ + THREAD_COMPLETED_OR_FAILURE_HANDLED = "THREAD_COMPLETED_OR_FAILURE_HANDLED", + /** + * THREAD_UNSUCCESSFUL - The ThreadRun did not complete successfully, and there wasn't a successful + * run of a Failure Handler for the Failure that was thrown. + */ + THREAD_UNSUCCESSFUL = "THREAD_UNSUCCESSFUL", + UNRECOGNIZED = "UNRECOGNIZED", +} + +export function waitForThreadsRun_WaitingThreadStatusFromJSON(object: any): WaitForThreadsRun_WaitingThreadStatus { + switch (object) { + case 0: + case "THREAD_IN_PROGRESS": + return WaitForThreadsRun_WaitingThreadStatus.THREAD_IN_PROGRESS; + case 1: + case "THREAD_HANDLING_FAILURE": + return WaitForThreadsRun_WaitingThreadStatus.THREAD_HANDLING_FAILURE; + case 2: + case "THREAD_COMPLETED_OR_FAILURE_HANDLED": + return WaitForThreadsRun_WaitingThreadStatus.THREAD_COMPLETED_OR_FAILURE_HANDLED; + case 3: + case "THREAD_UNSUCCESSFUL": + return WaitForThreadsRun_WaitingThreadStatus.THREAD_UNSUCCESSFUL; + case -1: + case "UNRECOGNIZED": + default: + return WaitForThreadsRun_WaitingThreadStatus.UNRECOGNIZED; + } +} + +export function waitForThreadsRun_WaitingThreadStatusToJSON(object: WaitForThreadsRun_WaitingThreadStatus): string { + switch (object) { + case WaitForThreadsRun_WaitingThreadStatus.THREAD_IN_PROGRESS: + return "THREAD_IN_PROGRESS"; + case WaitForThreadsRun_WaitingThreadStatus.THREAD_HANDLING_FAILURE: + return "THREAD_HANDLING_FAILURE"; + case WaitForThreadsRun_WaitingThreadStatus.THREAD_COMPLETED_OR_FAILURE_HANDLED: + return "THREAD_COMPLETED_OR_FAILURE_HANDLED"; + case WaitForThreadsRun_WaitingThreadStatus.THREAD_UNSUCCESSFUL: + return "THREAD_UNSUCCESSFUL"; + case WaitForThreadsRun_WaitingThreadStatus.UNRECOGNIZED: + default: + return "UNRECOGNIZED"; + } +} + +export function waitForThreadsRun_WaitingThreadStatusToNumber(object: WaitForThreadsRun_WaitingThreadStatus): number { + switch (object) { + case WaitForThreadsRun_WaitingThreadStatus.THREAD_IN_PROGRESS: + return 0; + case WaitForThreadsRun_WaitingThreadStatus.THREAD_HANDLING_FAILURE: + return 1; + case WaitForThreadsRun_WaitingThreadStatus.THREAD_COMPLETED_OR_FAILURE_HANDLED: + return 2; + case WaitForThreadsRun_WaitingThreadStatus.THREAD_UNSUCCESSFUL: + return 3; + case WaitForThreadsRun_WaitingThreadStatus.UNRECOGNIZED: + default: + return -1; + } } /** A 'WaitForThread' structure defines a thread that is being waited for. */ @@ -177,8 +239,17 @@ export interface WaitForThreadsRun_WaitForThread { threadStatus: LHStatus; /** The number of the ThreadRun being waited for. */ threadRunNumber: number; - /** INTERNAL: flag used by scheduler internally. */ - alreadyHandled: boolean; + /** + * The "waiting status" of this specific thread: whether it's still running, + * already done, handling a failure, or completely failed. + */ + waitingStatus: WaitForThreadsRun_WaitingThreadStatus; + /** + * If there is a failure on the ThreadRun, and we have a failure handler defined + * for it, then we will start a failure handler for this threadrun. This field + * is the id of that threadRun. + */ + failureHandlerThreadRunId?: number | undefined; } /** The sub-node structure for an EXTERNAL_EVENT NodeRun. */ @@ -949,7 +1020,7 @@ export const StartMultipleThreadsRun = { }; function createBaseWaitForThreadsRun(): WaitForThreadsRun { - return { threads: [], policy: WaitForThreadsPolicy.STOP_ON_FAILURE }; + return { threads: [] }; } export const WaitForThreadsRun = { @@ -957,9 +1028,6 @@ export const WaitForThreadsRun = { for (const v of message.threads) { WaitForThreadsRun_WaitForThread.encode(v!, writer.uint32(10).fork()).ldelim(); } - if (message.policy !== WaitForThreadsPolicy.STOP_ON_FAILURE) { - writer.uint32(16).int32(waitForThreadsPolicyToNumber(message.policy)); - } return writer; }, @@ -977,13 +1045,6 @@ export const WaitForThreadsRun = { message.threads.push(WaitForThreadsRun_WaitForThread.decode(reader, reader.uint32())); continue; - case 2: - if (tag !== 16) { - break; - } - - message.policy = waitForThreadsPolicyFromJSON(reader.int32()); - continue; } if ((tag & 7) === 4 || tag === 0) { break; @@ -998,7 +1059,6 @@ export const WaitForThreadsRun = { threads: globalThis.Array.isArray(object?.threads) ? object.threads.map((e: any) => WaitForThreadsRun_WaitForThread.fromJSON(e)) : [], - policy: isSet(object.policy) ? waitForThreadsPolicyFromJSON(object.policy) : WaitForThreadsPolicy.STOP_ON_FAILURE, }; }, @@ -1007,9 +1067,6 @@ export const WaitForThreadsRun = { if (message.threads?.length) { obj.threads = message.threads.map((e) => WaitForThreadsRun_WaitForThread.toJSON(e)); } - if (message.policy !== WaitForThreadsPolicy.STOP_ON_FAILURE) { - obj.policy = waitForThreadsPolicyToJSON(message.policy); - } return obj; }, @@ -1019,13 +1076,18 @@ export const WaitForThreadsRun = { fromPartial, I>>(object: I): WaitForThreadsRun { const message = createBaseWaitForThreadsRun(); message.threads = object.threads?.map((e) => WaitForThreadsRun_WaitForThread.fromPartial(e)) || []; - message.policy = object.policy ?? WaitForThreadsPolicy.STOP_ON_FAILURE; return message; }, }; function createBaseWaitForThreadsRun_WaitForThread(): WaitForThreadsRun_WaitForThread { - return { threadEndTime: undefined, threadStatus: LHStatus.STARTING, threadRunNumber: 0, alreadyHandled: false }; + return { + threadEndTime: undefined, + threadStatus: LHStatus.STARTING, + threadRunNumber: 0, + waitingStatus: WaitForThreadsRun_WaitingThreadStatus.THREAD_IN_PROGRESS, + failureHandlerThreadRunId: undefined, + }; } export const WaitForThreadsRun_WaitForThread = { @@ -1039,8 +1101,11 @@ export const WaitForThreadsRun_WaitForThread = { if (message.threadRunNumber !== 0) { writer.uint32(24).int32(message.threadRunNumber); } - if (message.alreadyHandled === true) { - writer.uint32(40).bool(message.alreadyHandled); + if (message.waitingStatus !== WaitForThreadsRun_WaitingThreadStatus.THREAD_IN_PROGRESS) { + writer.uint32(32).int32(waitForThreadsRun_WaitingThreadStatusToNumber(message.waitingStatus)); + } + if (message.failureHandlerThreadRunId !== undefined) { + writer.uint32(40).int32(message.failureHandlerThreadRunId); } return writer; }, @@ -1073,12 +1138,19 @@ export const WaitForThreadsRun_WaitForThread = { message.threadRunNumber = reader.int32(); continue; + case 4: + if (tag !== 32) { + break; + } + + message.waitingStatus = waitForThreadsRun_WaitingThreadStatusFromJSON(reader.int32()); + continue; case 5: if (tag !== 40) { break; } - message.alreadyHandled = reader.bool(); + message.failureHandlerThreadRunId = reader.int32(); continue; } if ((tag & 7) === 4 || tag === 0) { @@ -1094,7 +1166,12 @@ export const WaitForThreadsRun_WaitForThread = { threadEndTime: isSet(object.threadEndTime) ? globalThis.String(object.threadEndTime) : undefined, threadStatus: isSet(object.threadStatus) ? lHStatusFromJSON(object.threadStatus) : LHStatus.STARTING, threadRunNumber: isSet(object.threadRunNumber) ? globalThis.Number(object.threadRunNumber) : 0, - alreadyHandled: isSet(object.alreadyHandled) ? globalThis.Boolean(object.alreadyHandled) : false, + waitingStatus: isSet(object.waitingStatus) + ? waitForThreadsRun_WaitingThreadStatusFromJSON(object.waitingStatus) + : WaitForThreadsRun_WaitingThreadStatus.THREAD_IN_PROGRESS, + failureHandlerThreadRunId: isSet(object.failureHandlerThreadRunId) + ? globalThis.Number(object.failureHandlerThreadRunId) + : undefined, }; }, @@ -1109,8 +1186,11 @@ export const WaitForThreadsRun_WaitForThread = { if (message.threadRunNumber !== 0) { obj.threadRunNumber = Math.round(message.threadRunNumber); } - if (message.alreadyHandled === true) { - obj.alreadyHandled = message.alreadyHandled; + if (message.waitingStatus !== WaitForThreadsRun_WaitingThreadStatus.THREAD_IN_PROGRESS) { + obj.waitingStatus = waitForThreadsRun_WaitingThreadStatusToJSON(message.waitingStatus); + } + if (message.failureHandlerThreadRunId !== undefined) { + obj.failureHandlerThreadRunId = Math.round(message.failureHandlerThreadRunId); } return obj; }, @@ -1125,7 +1205,8 @@ export const WaitForThreadsRun_WaitForThread = { message.threadEndTime = object.threadEndTime ?? undefined; message.threadStatus = object.threadStatus ?? LHStatus.STARTING; message.threadRunNumber = object.threadRunNumber ?? 0; - message.alreadyHandled = object.alreadyHandled ?? false; + message.waitingStatus = object.waitingStatus ?? WaitForThreadsRun_WaitingThreadStatus.THREAD_IN_PROGRESS; + message.failureHandlerThreadRunId = object.failureHandlerThreadRunId ?? undefined; return message; }, }; diff --git a/dashboard/apps/web/littlehorse-public-api/wf_spec.ts b/dashboard/apps/web/littlehorse-public-api/wf_spec.ts index 070417209..28f8c10d3 100644 --- a/dashboard/apps/web/littlehorse-public-api/wf_spec.ts +++ b/dashboard/apps/web/littlehorse-public-api/wf_spec.ts @@ -10,10 +10,6 @@ import { variableTypeFromJSON, variableTypeToJSON, variableTypeToNumber, - WaitForThreadsPolicy, - waitForThreadsPolicyFromJSON, - waitForThreadsPolicyToJSON, - waitForThreadsPolicyToNumber, } from "./common_enums"; import { Comparator, @@ -251,19 +247,19 @@ export function failureHandlerDef_LHFailureTypeToNumber(object: FailureHandlerDe } export interface WaitForThreadsNode { - /** - * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and - * for compatibility reasons, we cannot wrap it into a separate message. - */ - threads: WaitForThreadsNode_ThreadToWaitFor[]; + threads?: WaitForThreadsNode_ThreadsToWaitFor | undefined; threadList?: VariableAssignment | undefined; - policy: WaitForThreadsPolicy; + perThreadFailureHandlers: FailureHandlerDef[]; } export interface WaitForThreadsNode_ThreadToWaitFor { threadRunNumber: VariableAssignment | undefined; } +export interface WaitForThreadsNode_ThreadsToWaitFor { + threads: WaitForThreadsNode_ThreadToWaitFor[]; +} + export interface ExternalEventNode { externalEventDefId: ExternalEventDefId | undefined; timeoutSeconds: VariableAssignment | undefined; @@ -1867,19 +1863,19 @@ export const FailureHandlerDef = { }; function createBaseWaitForThreadsNode(): WaitForThreadsNode { - return { threads: [], threadList: undefined, policy: WaitForThreadsPolicy.STOP_ON_FAILURE }; + return { threads: undefined, threadList: undefined, perThreadFailureHandlers: [] }; } export const WaitForThreadsNode = { encode(message: WaitForThreadsNode, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { - for (const v of message.threads) { - WaitForThreadsNode_ThreadToWaitFor.encode(v!, writer.uint32(10).fork()).ldelim(); + if (message.threads !== undefined) { + WaitForThreadsNode_ThreadsToWaitFor.encode(message.threads, writer.uint32(10).fork()).ldelim(); } if (message.threadList !== undefined) { - VariableAssignment.encode(message.threadList, writer.uint32(26).fork()).ldelim(); + VariableAssignment.encode(message.threadList, writer.uint32(18).fork()).ldelim(); } - if (message.policy !== WaitForThreadsPolicy.STOP_ON_FAILURE) { - writer.uint32(16).int32(waitForThreadsPolicyToNumber(message.policy)); + for (const v of message.perThreadFailureHandlers) { + FailureHandlerDef.encode(v!, writer.uint32(26).fork()).ldelim(); } return writer; }, @@ -1896,21 +1892,21 @@ export const WaitForThreadsNode = { break; } - message.threads.push(WaitForThreadsNode_ThreadToWaitFor.decode(reader, reader.uint32())); + message.threads = WaitForThreadsNode_ThreadsToWaitFor.decode(reader, reader.uint32()); continue; - case 3: - if (tag !== 26) { + case 2: + if (tag !== 18) { break; } message.threadList = VariableAssignment.decode(reader, reader.uint32()); continue; - case 2: - if (tag !== 16) { + case 3: + if (tag !== 26) { break; } - message.policy = waitForThreadsPolicyFromJSON(reader.int32()); + message.perThreadFailureHandlers.push(FailureHandlerDef.decode(reader, reader.uint32())); continue; } if ((tag & 7) === 4 || tag === 0) { @@ -1923,24 +1919,24 @@ export const WaitForThreadsNode = { fromJSON(object: any): WaitForThreadsNode { return { - threads: globalThis.Array.isArray(object?.threads) - ? object.threads.map((e: any) => WaitForThreadsNode_ThreadToWaitFor.fromJSON(e)) - : [], + threads: isSet(object.threads) ? WaitForThreadsNode_ThreadsToWaitFor.fromJSON(object.threads) : undefined, threadList: isSet(object.threadList) ? VariableAssignment.fromJSON(object.threadList) : undefined, - policy: isSet(object.policy) ? waitForThreadsPolicyFromJSON(object.policy) : WaitForThreadsPolicy.STOP_ON_FAILURE, + perThreadFailureHandlers: globalThis.Array.isArray(object?.perThreadFailureHandlers) + ? object.perThreadFailureHandlers.map((e: any) => FailureHandlerDef.fromJSON(e)) + : [], }; }, toJSON(message: WaitForThreadsNode): unknown { const obj: any = {}; - if (message.threads?.length) { - obj.threads = message.threads.map((e) => WaitForThreadsNode_ThreadToWaitFor.toJSON(e)); + if (message.threads !== undefined) { + obj.threads = WaitForThreadsNode_ThreadsToWaitFor.toJSON(message.threads); } if (message.threadList !== undefined) { obj.threadList = VariableAssignment.toJSON(message.threadList); } - if (message.policy !== WaitForThreadsPolicy.STOP_ON_FAILURE) { - obj.policy = waitForThreadsPolicyToJSON(message.policy); + if (message.perThreadFailureHandlers?.length) { + obj.perThreadFailureHandlers = message.perThreadFailureHandlers.map((e) => FailureHandlerDef.toJSON(e)); } return obj; }, @@ -1950,11 +1946,14 @@ export const WaitForThreadsNode = { }, fromPartial, I>>(object: I): WaitForThreadsNode { const message = createBaseWaitForThreadsNode(); - message.threads = object.threads?.map((e) => WaitForThreadsNode_ThreadToWaitFor.fromPartial(e)) || []; + message.threads = (object.threads !== undefined && object.threads !== null) + ? WaitForThreadsNode_ThreadsToWaitFor.fromPartial(object.threads) + : undefined; message.threadList = (object.threadList !== undefined && object.threadList !== null) ? VariableAssignment.fromPartial(object.threadList) : undefined; - message.policy = object.policy ?? WaitForThreadsPolicy.STOP_ON_FAILURE; + message.perThreadFailureHandlers = object.perThreadFailureHandlers?.map((e) => FailureHandlerDef.fromPartial(e)) || + []; return message; }, }; @@ -2024,6 +2023,71 @@ export const WaitForThreadsNode_ThreadToWaitFor = { }, }; +function createBaseWaitForThreadsNode_ThreadsToWaitFor(): WaitForThreadsNode_ThreadsToWaitFor { + return { threads: [] }; +} + +export const WaitForThreadsNode_ThreadsToWaitFor = { + encode(message: WaitForThreadsNode_ThreadsToWaitFor, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer { + for (const v of message.threads) { + WaitForThreadsNode_ThreadToWaitFor.encode(v!, writer.uint32(10).fork()).ldelim(); + } + return writer; + }, + + decode(input: _m0.Reader | Uint8Array, length?: number): WaitForThreadsNode_ThreadsToWaitFor { + const reader = input instanceof _m0.Reader ? input : _m0.Reader.create(input); + let end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseWaitForThreadsNode_ThreadsToWaitFor(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + if (tag !== 10) { + break; + } + + message.threads.push(WaitForThreadsNode_ThreadToWaitFor.decode(reader, reader.uint32())); + continue; + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skipType(tag & 7); + } + return message; + }, + + fromJSON(object: any): WaitForThreadsNode_ThreadsToWaitFor { + return { + threads: globalThis.Array.isArray(object?.threads) + ? object.threads.map((e: any) => WaitForThreadsNode_ThreadToWaitFor.fromJSON(e)) + : [], + }; + }, + + toJSON(message: WaitForThreadsNode_ThreadsToWaitFor): unknown { + const obj: any = {}; + if (message.threads?.length) { + obj.threads = message.threads.map((e) => WaitForThreadsNode_ThreadToWaitFor.toJSON(e)); + } + return obj; + }, + + create, I>>( + base?: I, + ): WaitForThreadsNode_ThreadsToWaitFor { + return WaitForThreadsNode_ThreadsToWaitFor.fromPartial(base ?? ({} as any)); + }, + fromPartial, I>>( + object: I, + ): WaitForThreadsNode_ThreadsToWaitFor { + const message = createBaseWaitForThreadsNode_ThreadsToWaitFor(); + message.threads = object.threads?.map((e) => WaitForThreadsNode_ThreadToWaitFor.fromPartial(e)) || []; + return message; + }, +}; + function createBaseExternalEventNode(): ExternalEventNode { return { externalEventDefId: undefined, timeoutSeconds: undefined }; } diff --git a/e2e-tests/src/main/java/io/littlehorse/tests/cases/workflow/AWChildThreadExceptionHandler.java b/e2e-tests/src/main/java/io/littlehorse/tests/cases/workflow/AWChildThreadExceptionHandler.java deleted file mode 100644 index fd12f894d..000000000 --- a/e2e-tests/src/main/java/io/littlehorse/tests/cases/workflow/AWChildThreadExceptionHandler.java +++ /dev/null @@ -1,85 +0,0 @@ -package io.littlehorse.tests.cases.workflow; - -import io.littlehorse.sdk.common.config.LHConfig; -import io.littlehorse.sdk.common.proto.LHStatus; -import io.littlehorse.sdk.common.proto.LittleHorseGrpc.LittleHorseBlockingStub; -import io.littlehorse.sdk.common.proto.WaitForThreadsPolicy; -import io.littlehorse.sdk.wfsdk.NodeOutput; -import io.littlehorse.sdk.wfsdk.SpawnedThread; -import io.littlehorse.sdk.wfsdk.SpawnedThreads; -import io.littlehorse.sdk.wfsdk.Workflow; -import io.littlehorse.sdk.wfsdk.internal.WorkflowImpl; -import io.littlehorse.sdk.worker.LHTaskMethod; -import io.littlehorse.tests.TestFailure; -import io.littlehorse.tests.WorkflowLogicTest; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; - -public class AWChildThreadExceptionHandler extends WorkflowLogicTest { - - public AWChildThreadExceptionHandler(LittleHorseBlockingStub client, LHConfig workerConfig) { - super(client, workerConfig); - } - - public String getDescription() { - return ("Tests that we can put an exception handler on WAIT_FOR_THREAD node" - + " in case the child thread fails."); - } - - public Workflow getWorkflowImpl() { - return new WorkflowImpl(getWorkflowName(), thread -> { - SpawnedThread childThread = thread.spawnThread( - child -> { - child.execute("aw-fail"); - }, - "child", - null); - - NodeOutput toHandle = thread.waitForThreads(SpawnedThreads.of(childThread)) - .withPolicy(WaitForThreadsPolicy.STOP_ON_FAILURE); - - thread.handleError(toHandle, handler -> { - handler.execute("aw-echo", "hi from handler"); - }); - thread.execute("aw-succeed"); - }); - } - - public List getTaskWorkerObjects() { - return Arrays.asList(new AWSimpleTask()); - } - - public List launchAndCheckWorkflows(LittleHorseBlockingStub client) - throws TestFailure, InterruptedException, IOException { - String wfRunId = runWf(client); - - Thread.sleep(500); - assertStatus(client, wfRunId, LHStatus.COMPLETED); - assertThreadStatus(client, wfRunId, 0, LHStatus.COMPLETED); - assertThreadStatus(client, wfRunId, 1, LHStatus.ERROR); - assertThreadStatus(client, wfRunId, 2, LHStatus.COMPLETED); - assertTaskOutputsMatch(client, wfRunId, 2, "hi from handler"); - assertTaskOutputsMatch(client, wfRunId, 0, "Success!"); - - return Arrays.asList(wfRunId); - } -} - -class AWSimpleTask { - - @LHTaskMethod("aw-fail") - public String fail() { - throw new RuntimeException("ooph"); - } - - @LHTaskMethod("aw-echo") - public String echo(String input) { - return input; - } - - @LHTaskMethod("aw-succeed") - public String succeed() { - return "Success!"; - } -} diff --git a/e2e-tests/src/main/java/io/littlehorse/tests/cases/workflow/BASleepUntilTimestamp.java b/e2e-tests/src/main/java/io/littlehorse/tests/cases/workflow/BASleepUntilTimestamp.java index a536bdeb2..85234c100 100644 --- a/e2e-tests/src/main/java/io/littlehorse/tests/cases/workflow/BASleepUntilTimestamp.java +++ b/e2e-tests/src/main/java/io/littlehorse/tests/cases/workflow/BASleepUntilTimestamp.java @@ -44,7 +44,7 @@ public List launchAndCheckWorkflows(LittleHorseBlockingStub client) client, Arg.of("timestamp-to-wait-for", new Date(System.currentTimeMillis() + (1000 * 2)).getTime())); Thread.sleep(1000); assertStatus(client, wfRunId, LHStatus.RUNNING); - Thread.sleep(7000); // note that the timer interval is 4 seconds + Thread.sleep(3000); // note that the timer interval is 1 second assertStatus(client, wfRunId, LHStatus.COMPLETED); return Arrays.asList(wfRunId); diff --git a/e2e-tests/src/main/java/io/littlehorse/tests/cases/workflow/BBWaitMultipleChildren.java b/e2e-tests/src/main/java/io/littlehorse/tests/cases/workflow/BBWaitMultipleChildren.java deleted file mode 100644 index 09b586d1f..000000000 --- a/e2e-tests/src/main/java/io/littlehorse/tests/cases/workflow/BBWaitMultipleChildren.java +++ /dev/null @@ -1,284 +0,0 @@ -package io.littlehorse.tests.cases.workflow; - -import io.littlehorse.sdk.common.LHLibUtil; -import io.littlehorse.sdk.common.config.LHConfig; -import io.littlehorse.sdk.common.exception.LHSerdeError; -import io.littlehorse.sdk.common.proto.LHStatus; -import io.littlehorse.sdk.common.proto.LittleHorseGrpc.LittleHorseBlockingStub; -import io.littlehorse.sdk.common.proto.NodeRun; -import io.littlehorse.sdk.common.proto.PutExternalEventRequest; -import io.littlehorse.sdk.common.proto.ThreadRun; -import io.littlehorse.sdk.common.proto.VariableMutationType; -import io.littlehorse.sdk.common.proto.VariableType; -import io.littlehorse.sdk.common.proto.VariableValue; -import io.littlehorse.sdk.common.proto.WaitForThreadsPolicy; -import io.littlehorse.sdk.common.proto.WaitForThreadsRun; -import io.littlehorse.sdk.common.proto.WfRun; -import io.littlehorse.sdk.wfsdk.SpawnedThread; -import io.littlehorse.sdk.wfsdk.SpawnedThreads; -import io.littlehorse.sdk.wfsdk.WfRunVariable; -import io.littlehorse.sdk.wfsdk.Workflow; -import io.littlehorse.sdk.wfsdk.WorkflowThread; -import io.littlehorse.sdk.wfsdk.internal.WorkflowImpl; -import io.littlehorse.sdk.worker.LHTaskMethod; -import io.littlehorse.tests.TestFailure; -import io.littlehorse.tests.WorkflowLogicTest; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -// We don't yet add a test to see what happens when we try to wait for non-existent -// ThreadRun, because it's not really possible to fall into that trap given the -// safeguards of the java wf sdk. But we might do that test in the future. -public class BBWaitMultipleChildren extends WorkflowLogicTest { - - public BBWaitMultipleChildren(LittleHorseBlockingStub client, LHConfig workerConfig) { - super(client, workerConfig); - } - - public String getDescription() { - return ("Tests happy path behavior of WAIT_FOR_THREADS node with " + "multiple threads to wait for."); - } - - /* - * This workflow basically spawns two child threads and waits for them. - * Each child thread listens for an external event, then executes a - * task after that event has come through. - * - * The Child workflows implicitly assume that the external event is a - * JSON_OBJ with structhre `{"myInt": }`, so we can use - * that to make the child workflows fail in order to test certain edge - * cases. - * - * Additionally, we can use the external event to control the order in - * which the child threads complete, to verify that when the children - * are in progress that the status is properly reflected in the workflow. - */ - public Workflow getWorkflowImpl() { - return new WorkflowImpl(getWorkflowName(), thread -> { - SpawnedThread child1 = thread.spawnThread(this::thread1, "child-1", Map.of("input1", Map.of())); - SpawnedThread child2 = thread.spawnThread(this::thread2, "child-2", Map.of("input2", Map.of())); - - thread.waitForThreads(SpawnedThreads.of(child1, child2)).withPolicy(WaitForThreadsPolicy.STOP_ON_FAILURE); - }); - } - - private void thread1(WorkflowThread thread) { - // We use this variable so that we can control whether the thread - // fails by manipulating the external event content - WfRunVariable eventOutput = thread.addVariable("input1", VariableType.JSON_OBJ); - thread.mutate(eventOutput, VariableMutationType.ASSIGN, thread.waitForEvent("thread-1-event")); - thread.execute("add-1", eventOutput.jsonPath("$.myInt")); - } - - private void thread2(WorkflowThread thread) { - // We use this variable so that we can control whether the thread - // fails by manipulating the external event content - WfRunVariable eventOutput = thread.addVariable("input2", VariableType.JSON_OBJ); - thread.mutate(eventOutput, VariableMutationType.ASSIGN, thread.waitForEvent("thread-2-event")); - thread.execute("add-1", eventOutput.jsonPath("$.myInt")); - } - - public List getTaskWorkerObjects() { - return Arrays.asList(new BBSimpleTask()); - } - - public List launchAndCheckWorkflows(LittleHorseBlockingStub client) - throws TestFailure, InterruptedException, IOException { - return Arrays.asList(happyPath(client), firstThreadFails(client), secondThreadFails(client)); - } - - private VariableValue generateEvent(String key, Object value) throws TestFailure { - try { - return LHLibUtil.objToVarVal(Map.of(key, value)); - } catch (LHSerdeError exn) { - throw new TestFailure(this, "impossible: serde error: " + exn.getMessage()); - } - } - - /* - * This tests the happy path case where thread 1 finishes first. - */ - private String happyPath(LittleHorseBlockingStub client) throws TestFailure, InterruptedException, IOException { - String id = runWf(null, client); - assertStatus(client, id, LHStatus.RUNNING); - - // Now make sure there are two child threads. - WfRun wfRun = getWfRun(client, id); - assertThat(wfRun.getThreadRunsCount() == 3, "WfRun " + id + " should have three threadruns"); - NodeRun waitForThreadNode = getNodeRun(client, id, 0, 3); - assertThat( - waitForThreadNode.getStatus() == LHStatus.RUNNING, - "Wait for thread node should still be running! Wf: " + id); - - WaitForThreadsRun waitingThreads = waitForThreadNode.getWaitThreads(); - assertThat( - waitingThreads.getThreadsCount() == 2, - "Wait for thread node should have two threads it's waiting on! Wf: " + id); - assertThat( - waitingThreads.getThreads(0).getThreadRunNumber() == 1, - "Expected first waiting thread to be thread 1! Wf: " + id); - assertThat( - waitingThreads.getThreads(1).getThreadRunNumber() == 2, - "Expected second waiting thread to be thread 2! Wf: " + id); - - // Now we make the first thread finish - ThreadRun thread1 = wfRun.getThreadRuns(1); - assertThat(thread1.getStatus() == LHStatus.RUNNING, "Thread1 should be running! Wf: " + id); - client.putExternalEvent(PutExternalEventRequest.newBuilder() - .setContent(generateEvent("myInt", 5)) - .setWfRunId(LHLibUtil.wfRunId(id)) - .setExternalEventDefId(LHLibUtil.externalEventDefId("thread-1-event")) - .build()); - // There is a Task Node after the External Event Node, so we wait - // for it to finish. - Thread.sleep(200); - wfRun = getWfRun(client, id); - assertThat(wfRun.getStatus() == LHStatus.RUNNING, "WfRun should still be running! Wf: " + id); - thread1 = wfRun.getThreadRuns(1); - assertThat(thread1.getStatus() == LHStatus.COMPLETED, "Thread1 should be Done! Wf: " + id); - ThreadRun entrypoint = wfRun.getThreadRuns(0); - assertThat( - entrypoint.getStatus() == LHStatus.RUNNING, - "Entrypoint should be running becasue thread2 not done yet! Wf:" + id); - waitingThreads = getNodeRun(client, id, 0, 3).getWaitThreads(); - assertThat( - waitingThreads.getThreads(0).getThreadStatus() == LHStatus.COMPLETED, - "Should have noticed that the first thread finished! WF: " + id); - assertThat( - waitingThreads.getThreads(1).getThreadStatus() == LHStatus.RUNNING, - "Second thread should still be running! WF: " + id); - - // Now we complete the second thread. - client.putExternalEvent(PutExternalEventRequest.newBuilder() - .setContent(generateEvent("myInt", 10)) - .setWfRunId(LHLibUtil.wfRunId(id)) - .setExternalEventDefId(LHLibUtil.externalEventDefId("thread-2-event")) - .build()); - Thread.sleep(200); - wfRun = getWfRun(client, id); - assertThat(wfRun.getStatus() == LHStatus.COMPLETED, "WfRun should still be completed now! Wf: " + id); - waitingThreads = getNodeRun(client, id, 0, 3).getWaitThreads(); - assertThat( - waitingThreads.getThreads(1).getThreadStatus() == LHStatus.COMPLETED, - "Should have noticed that the second thread finished! WF: " + id); - return id; - } - - private String firstThreadFails(LittleHorseBlockingStub client) - throws TestFailure, InterruptedException, IOException { - String id = runWf(null, client); - assertStatus(client, id, LHStatus.RUNNING); - - // Now we make the first thread fail by sending a malformed event (recall - // that the child thread expects $.myInt to be an int, not a string) - client.putExternalEvent(PutExternalEventRequest.newBuilder() - .setContent(generateEvent("myInt", "some-string")) - .setWfRunId(LHLibUtil.wfRunId(id)) - .setExternalEventDefId(LHLibUtil.externalEventDefId("thread-1-event")) - .build()); - - // There is a Task Node after the External Event Node, so we wait - // for it to finish. - Thread.sleep(200); - WfRun wfRun = getWfRun(client, id); - System.out.println(wfRun.getStatus()); - assertThat(wfRun.getStatus() == LHStatus.ERROR, "WfRun should still be running! Wf: " + id); - ThreadRun thread1 = wfRun.getThreadRuns(1); - assertThat(thread1.getStatus() == LHStatus.ERROR, "Thread1 should have failed! Wf: " + id); - - NodeRun failedNode = getNodeRun(client, id, 1, 2); - assertThat( - failedNode.getStatus() == LHStatus.ERROR, - "The task node of thread 1 should have failed with " + "TASK_INPUT_VAR_SUB_ERROR, wf: " + id); - - // This line here checks that the NodeRun failed due to input var sub error. - assertThat( - !failedNode.getTask().hasTaskRunId(), - "The TaskRun should not have been created because the input vars" + " were invalid! Wf: " + id); - assertThat(failedNode.getFailuresCount() == 1, "The failed node should have 1 failure. Wf: " + id); - assertThat( - failedNode.getFailures(0).getFailureName().equals("VAR_SUB_ERROR"), - "The node should have failed with 'VAR_SUB_ERROR'! Wf: " + id); - - // Now we check that the ThreadRun was noticed as failed by the WaitThreadRun - WaitForThreadsRun waitingThreads = getNodeRun(client, id, 0, 3).getWaitThreads(); - assertThat( - waitingThreads.getThreads(0).getThreadStatus() == LHStatus.ERROR, - "Should have noticed that the first thread Died! WF: " + id); - assertThat( - waitingThreads.getThreads(1).getThreadStatus() == LHStatus.HALTED, - "Second thread should still be running! WF: " + id); - - // Now we complete the second thread. - client.putExternalEvent(PutExternalEventRequest.newBuilder() - .setContent(generateEvent("myInt", 10)) - .setWfRunId(LHLibUtil.wfRunId(id)) - .setExternalEventDefId(LHLibUtil.externalEventDefId("thread-2-event")) - .build()); - Thread.sleep(200); - wfRun = getWfRun(client, id); - assertThat( - wfRun.getStatus() == LHStatus.ERROR, "WfRun should have failed because first thread died! Wf: " + id); - waitingThreads = getNodeRun(client, id, 0, 3).getWaitThreads(); - assertThat( - waitingThreads.getThreads(1).getThreadStatus() == LHStatus.HALTED, - "Should have noticed that the second thread finished! WF: " + id); - return id; - } - - private String secondThreadFails(LittleHorseBlockingStub client) - throws TestFailure, InterruptedException, IOException { - String id = runWf(null, client); - assertStatus(client, id, LHStatus.RUNNING); - - // Now we make the first thread complete - client.putExternalEvent(PutExternalEventRequest.newBuilder() - .setContent(generateEvent("myInt", 137)) - .setWfRunId(LHLibUtil.wfRunId(id)) - .setExternalEventDefId(LHLibUtil.externalEventDefId("thread-1-event")) - .build()); - - // There is a Task Node after the External Event Node, so we wait - // for it to finish. - Thread.sleep(200); - WfRun wfRun = getWfRun(client, id); - assertThat(wfRun.getStatus() == LHStatus.RUNNING, "WfRun should still be running! Wf: " + id); - ThreadRun thread1 = wfRun.getThreadRuns(1); - assertThat(thread1.getStatus() == LHStatus.COMPLETED, "Thread1 should have finished! Wf: " + id); - - // Now we check that the ThreadRun was noticed as failed by the WaitThreadRun - WaitForThreadsRun waitingThreads = getNodeRun(client, id, 0, 3).getWaitThreads(); - assertThat( - waitingThreads.getThreads(0).getThreadStatus() == LHStatus.COMPLETED, - "Should have noticed that the first thread Died! WF: " + id); - assertThat( - waitingThreads.getThreads(1).getThreadStatus() == LHStatus.RUNNING, - "Second thread should still be running! WF: " + id); - - // Now we fail the second thread. - client.putExternalEvent(PutExternalEventRequest.newBuilder() - .setContent(generateEvent("myInt", "not-an-integer")) - .setWfRunId(LHLibUtil.wfRunId(id)) - .setExternalEventDefId(LHLibUtil.externalEventDefId("thread-2-event")) - .build()); - Thread.sleep(200); - wfRun = getWfRun(client, id); - assertThat( - wfRun.getStatus() == LHStatus.ERROR, "WfRun should have failed because second thread died! Wf: " + id); - waitingThreads = getNodeRun(client, id, 0, 3).getWaitThreads(); - assertThat( - waitingThreads.getThreads(1).getThreadStatus() == LHStatus.ERROR, - "Should have noticed that the second thread failed! WF: " + id); - return id; - } -} - -class BBSimpleTask { - - @LHTaskMethod("add-1") - public int addOne(int input) { - return input + 1; - } -} diff --git a/e2e-tests/src/main/java/io/littlehorse/tests/cases/workflow/BCChildFinishesFirst.java b/e2e-tests/src/main/java/io/littlehorse/tests/cases/workflow/BCChildFinishesFirst.java index cd2bf42da..27916aca2 100644 --- a/e2e-tests/src/main/java/io/littlehorse/tests/cases/workflow/BCChildFinishesFirst.java +++ b/e2e-tests/src/main/java/io/littlehorse/tests/cases/workflow/BCChildFinishesFirst.java @@ -3,7 +3,6 @@ import io.littlehorse.sdk.common.config.LHConfig; import io.littlehorse.sdk.common.proto.LHStatus; import io.littlehorse.sdk.common.proto.LittleHorseGrpc.LittleHorseBlockingStub; -import io.littlehorse.sdk.common.proto.WaitForThreadsPolicy; import io.littlehorse.sdk.wfsdk.SpawnedThread; import io.littlehorse.sdk.wfsdk.SpawnedThreads; import io.littlehorse.sdk.wfsdk.Workflow; @@ -47,7 +46,7 @@ public Workflow getWorkflowImpl() { return new WorkflowImpl(getWorkflowName(), thread -> { SpawnedThread child = thread.spawnThread(this::child, "child", null); thread.sleepSeconds(1); - thread.waitForThreads(SpawnedThreads.of(child)).withPolicy(WaitForThreadsPolicy.STOP_ON_FAILURE); + thread.waitForThreads(SpawnedThreads.of(child)); }); } diff --git a/examples/basic/src/main/java/io/littlehorse/examples/BasicExample.java b/examples/basic/src/main/java/io/littlehorse/examples/BasicExample.java index 9eeb06552..b69569ce2 100644 --- a/examples/basic/src/main/java/io/littlehorse/examples/BasicExample.java +++ b/examples/basic/src/main/java/io/littlehorse/examples/BasicExample.java @@ -23,10 +23,7 @@ public static Workflow getWorkflow() { return new WorkflowImpl( "example-basic", wf -> { - WfRunVariable theName = wf.addVariable( - "input-name", - VariableType.STR - ); + WfRunVariable theName = wf.addVariable("input-name", VariableType.STR).searchable(); wf.execute("greet", theName); } ); diff --git a/local-dev/configs/server-1.config b/local-dev/configs/server-1.config index 786e40a8d..f31a3acac 100644 --- a/local-dev/configs/server-1.config +++ b/local-dev/configs/server-1.config @@ -6,6 +6,7 @@ LHS_CLUSTER_PARTITIONS=12 LHS_STATE_DIR=/tmp/kafkaState LHS_STREAMS_NUM_WARMUP_REPLICAS=4 LHS_STREAMS_NUM_STANDBY_REPLICAS=0 +LHS_STREAMS_SESSION_TIMEOUT=10000 LHS_SHOULD_CREATE_TOPICS=true LHS_HEALTH_SERVICE_PORT=1822 diff --git a/schemas/common_enums.proto b/schemas/common_enums.proto index 31c2b5fd3..1528ae9be 100644 --- a/schemas/common_enums.proto +++ b/schemas/common_enums.proto @@ -105,7 +105,3 @@ enum LHErrorType { // An unexpected LittleHorse Internal error occurred. This is not expected to happen. INTERNAL_ERROR = 8; } - -enum WaitForThreadsPolicy{ - STOP_ON_FAILURE = 0; -} diff --git a/schemas/node_run.proto b/schemas/node_run.proto index b62bb797a..af09b5199 100644 --- a/schemas/node_run.proto +++ b/schemas/node_run.proto @@ -126,6 +126,25 @@ message StartMultipleThreadsRun { // The sub-node structure for a WAIT_FOR_THREADS NodeRun. message WaitForThreadsRun { + // The status of a single ThreadRun that we are waiting for. + enum WaitingThreadStatus { + // The ThreadRun is in progress (i.e. not COMPLETED nor EXCEPTION nor ERROR) + THREAD_IN_PROGRESS = 0; + + // The ThreadRun failed with some failure, and the FailureHandler is running + // for that Failure. + THREAD_HANDLING_FAILURE = 1; + + // We can mark this ThreadRun as "already waited for", meaning that either: + // 1. It completed successfully, OR + // 2. It failed, and the Failure Handler successfully completed + THREAD_COMPLETED_OR_FAILURE_HANDLED = 2; + + // The ThreadRun did not complete successfully, and there wasn't a successful + // run of a Failure Handler for the Failure that was thrown. + THREAD_UNSUCCESSFUL = 3; + } + // A 'WaitForThread' structure defines a thread that is being waited for. message WaitForThread { // The time at which the ThreadRun ended (successfully or not). Not set if the ThreadRun @@ -138,16 +157,18 @@ message WaitForThreadsRun { // The number of the ThreadRun being waited for. int32 thread_run_number = 3; - // INTERNAL: flag used by scheduler internally. - bool already_handled = 5; + // The "waiting status" of this specific thread: whether it's still running, + // already done, handling a failure, or completely failed. + WaitingThreadStatus waiting_status = 4; + + // If there is a failure on the ThreadRun, and we have a failure handler defined + // for it, then we will start a failure handler for this threadrun. This field + // is the id of that threadRun. + optional int32 failure_handler_thread_run_id = 5; } // The threads that are being waited for. repeated WaitForThread threads = 1; - - // The policy to use when handling failures for Threads. Currently, only - // one policy exists. - WaitForThreadsPolicy policy = 2; } // The sub-node structure for an EXTERNAL_EVENT NodeRun. diff --git a/schemas/wf_spec.proto b/schemas/wf_spec.proto index 3fa7fe720..b5d68612b 100644 --- a/schemas/wf_spec.proto +++ b/schemas/wf_spec.proto @@ -130,13 +130,15 @@ message WaitForThreadsNode { message ThreadToWaitFor { VariableAssignment thread_run_number = 1; } + message ThreadsToWaitFor { + repeated ThreadToWaitFor threads = 1; + } - // Either 1 or 3 is set. Cannot put `repeated` into a oneof, and - // for compatibility reasons, we cannot wrap it into a separate message. - repeated ThreadToWaitFor threads = 1; - optional VariableAssignment thread_list = 3; - - WaitForThreadsPolicy policy = 2; + oneof threads_to_wait_for { + ThreadsToWaitFor threads = 1; + VariableAssignment thread_list = 2; + } + repeated FailureHandlerDef per_thread_failure_handlers = 3; } message ExternalEventNode { diff --git a/sdk-go/common/model/common_enums.pb.go b/sdk-go/common/model/common_enums.pb.go index e8c9c7cb8..14461c218 100644 --- a/sdk-go/common/model/common_enums.pb.go +++ b/sdk-go/common/model/common_enums.pb.go @@ -408,49 +408,6 @@ func (LHErrorType) EnumDescriptor() ([]byte, []int) { return file_common_enums_proto_rawDescGZIP(), []int{5} } -type WaitForThreadsPolicy int32 - -const ( - WaitForThreadsPolicy_STOP_ON_FAILURE WaitForThreadsPolicy = 0 -) - -// Enum value maps for WaitForThreadsPolicy. -var ( - WaitForThreadsPolicy_name = map[int32]string{ - 0: "STOP_ON_FAILURE", - } - WaitForThreadsPolicy_value = map[string]int32{ - "STOP_ON_FAILURE": 0, - } -) - -func (x WaitForThreadsPolicy) Enum() *WaitForThreadsPolicy { - p := new(WaitForThreadsPolicy) - *p = x - return p -} - -func (x WaitForThreadsPolicy) String() string { - return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) -} - -func (WaitForThreadsPolicy) Descriptor() protoreflect.EnumDescriptor { - return file_common_enums_proto_enumTypes[6].Descriptor() -} - -func (WaitForThreadsPolicy) Type() protoreflect.EnumType { - return &file_common_enums_proto_enumTypes[6] -} - -func (x WaitForThreadsPolicy) Number() protoreflect.EnumNumber { - return protoreflect.EnumNumber(x) -} - -// Deprecated: Use WaitForThreadsPolicy.Descriptor instead. -func (WaitForThreadsPolicy) EnumDescriptor() ([]byte, []int) { - return file_common_enums_proto_rawDescGZIP(), []int{6} -} - var File_common_enums_proto protoreflect.FileDescriptor var file_common_enums_proto_rawDesc = []byte{ @@ -499,15 +456,12 @@ var file_common_enums_proto_rawDesc = []byte{ 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x05, 0x12, 0x0d, 0x0a, 0x09, 0x56, 0x41, 0x52, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x06, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x41, 0x53, 0x4b, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x07, 0x12, 0x12, 0x0a, 0x0e, 0x49, 0x4e, 0x54, - 0x45, 0x52, 0x4e, 0x41, 0x4c, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x08, 0x2a, 0x2b, 0x0a, - 0x14, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x50, - 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x13, 0x0a, 0x0f, 0x53, 0x54, 0x4f, 0x50, 0x5f, 0x4f, 0x4e, - 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x00, 0x42, 0x47, 0x0a, 0x1f, 0x69, 0x6f, - 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x73, 0x64, 0x6b, - 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, - 0x07, 0x2e, 0x3b, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0xaa, 0x02, 0x18, 0x4c, 0x69, 0x74, 0x74, 0x6c, - 0x65, 0x48, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x45, 0x52, 0x4e, 0x41, 0x4c, 0x5f, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x08, 0x42, 0x47, 0x0a, + 0x1f, 0x69, 0x6f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, + 0x73, 0x64, 0x6b, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x50, 0x01, 0x5a, 0x07, 0x2e, 0x3b, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0xaa, 0x02, 0x18, 0x4c, 0x69, + 0x74, 0x74, 0x6c, 0x65, 0x48, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, + 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -522,15 +476,14 @@ func file_common_enums_proto_rawDescGZIP() []byte { return file_common_enums_proto_rawDescData } -var file_common_enums_proto_enumTypes = make([]protoimpl.EnumInfo, 7) +var file_common_enums_proto_enumTypes = make([]protoimpl.EnumInfo, 6) var file_common_enums_proto_goTypes = []interface{}{ - (LHStatus)(0), // 0: littlehorse.LHStatus - (MetadataStatus)(0), // 1: littlehorse.MetadataStatus - (TaskStatus)(0), // 2: littlehorse.TaskStatus - (MetricsWindowLength)(0), // 3: littlehorse.MetricsWindowLength - (VariableType)(0), // 4: littlehorse.VariableType - (LHErrorType)(0), // 5: littlehorse.LHErrorType - (WaitForThreadsPolicy)(0), // 6: littlehorse.WaitForThreadsPolicy + (LHStatus)(0), // 0: littlehorse.LHStatus + (MetadataStatus)(0), // 1: littlehorse.MetadataStatus + (TaskStatus)(0), // 2: littlehorse.TaskStatus + (MetricsWindowLength)(0), // 3: littlehorse.MetricsWindowLength + (VariableType)(0), // 4: littlehorse.VariableType + (LHErrorType)(0), // 5: littlehorse.LHErrorType } var file_common_enums_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for method output_type @@ -550,7 +503,7 @@ func file_common_enums_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_common_enums_proto_rawDesc, - NumEnums: 7, + NumEnums: 6, NumMessages: 0, NumExtensions: 0, NumServices: 0, diff --git a/sdk-go/common/model/node_run.pb.go b/sdk-go/common/model/node_run.pb.go index d9f62d5fb..9ec76c74e 100644 --- a/sdk-go/common/model/node_run.pb.go +++ b/sdk-go/common/model/node_run.pb.go @@ -21,6 +21,67 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// The status of a single ThreadRun that we are waiting for. +type WaitForThreadsRun_WaitingThreadStatus int32 + +const ( + // The ThreadRun is in progress (i.e. not COMPLETED nor EXCEPTION nor ERROR) + WaitForThreadsRun_THREAD_IN_PROGRESS WaitForThreadsRun_WaitingThreadStatus = 0 + // The ThreadRun failed with some failure, and the FailureHandler is running + // for that Failure. + WaitForThreadsRun_THREAD_HANDLING_FAILURE WaitForThreadsRun_WaitingThreadStatus = 1 + // We can mark this ThreadRun as "already waited for", meaning that either: + // 1. It completed successfully, OR + // 2. It failed, and the Failure Handler successfully completed + WaitForThreadsRun_THREAD_COMPLETED_OR_FAILURE_HANDLED WaitForThreadsRun_WaitingThreadStatus = 2 + // The ThreadRun did not complete successfully, and there wasn't a successful + // run of a Failure Handler for the Failure that was thrown. + WaitForThreadsRun_THREAD_UNSUCCESSFUL WaitForThreadsRun_WaitingThreadStatus = 3 +) + +// Enum value maps for WaitForThreadsRun_WaitingThreadStatus. +var ( + WaitForThreadsRun_WaitingThreadStatus_name = map[int32]string{ + 0: "THREAD_IN_PROGRESS", + 1: "THREAD_HANDLING_FAILURE", + 2: "THREAD_COMPLETED_OR_FAILURE_HANDLED", + 3: "THREAD_UNSUCCESSFUL", + } + WaitForThreadsRun_WaitingThreadStatus_value = map[string]int32{ + "THREAD_IN_PROGRESS": 0, + "THREAD_HANDLING_FAILURE": 1, + "THREAD_COMPLETED_OR_FAILURE_HANDLED": 2, + "THREAD_UNSUCCESSFUL": 3, + } +) + +func (x WaitForThreadsRun_WaitingThreadStatus) Enum() *WaitForThreadsRun_WaitingThreadStatus { + p := new(WaitForThreadsRun_WaitingThreadStatus) + *p = x + return p +} + +func (x WaitForThreadsRun_WaitingThreadStatus) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (WaitForThreadsRun_WaitingThreadStatus) Descriptor() protoreflect.EnumDescriptor { + return file_node_run_proto_enumTypes[0].Descriptor() +} + +func (WaitForThreadsRun_WaitingThreadStatus) Type() protoreflect.EnumType { + return &file_node_run_proto_enumTypes[0] +} + +func (x WaitForThreadsRun_WaitingThreadStatus) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use WaitForThreadsRun_WaitingThreadStatus.Descriptor instead. +func (WaitForThreadsRun_WaitingThreadStatus) EnumDescriptor() ([]byte, []int) { + return file_node_run_proto_rawDescGZIP(), []int{7, 0} +} + // A NodeRun is a running instance of a Node in a ThreadRun. Note that a NodeRun // is a Getable object, meaning it can be retried from the LittleHorse grpc API. type NodeRun struct { @@ -606,9 +667,6 @@ type WaitForThreadsRun struct { // The threads that are being waited for. Threads []*WaitForThreadsRun_WaitForThread `protobuf:"bytes,1,rep,name=threads,proto3" json:"threads,omitempty"` - // The policy to use when handling failures for Threads. Currently, only - // one policy exists. - Policy WaitForThreadsPolicy `protobuf:"varint,2,opt,name=policy,proto3,enum=littlehorse.WaitForThreadsPolicy" json:"policy,omitempty"` } func (x *WaitForThreadsRun) Reset() { @@ -650,13 +708,6 @@ func (x *WaitForThreadsRun) GetThreads() []*WaitForThreadsRun_WaitForThread { return nil } -func (x *WaitForThreadsRun) GetPolicy() WaitForThreadsPolicy { - if x != nil { - return x.Policy - } - return WaitForThreadsPolicy_STOP_ON_FAILURE -} - // The sub-node structure for an EXTERNAL_EVENT NodeRun. type ExternalEventRun struct { state protoimpl.MessageState @@ -871,8 +922,13 @@ type WaitForThreadsRun_WaitForThread struct { ThreadStatus LHStatus `protobuf:"varint,2,opt,name=thread_status,json=threadStatus,proto3,enum=littlehorse.LHStatus" json:"thread_status,omitempty"` // The number of the ThreadRun being waited for. ThreadRunNumber int32 `protobuf:"varint,3,opt,name=thread_run_number,json=threadRunNumber,proto3" json:"thread_run_number,omitempty"` - // INTERNAL: flag used by scheduler internally. - AlreadyHandled bool `protobuf:"varint,5,opt,name=already_handled,json=alreadyHandled,proto3" json:"already_handled,omitempty"` + // The "waiting status" of this specific thread: whether it's still running, + // already done, handling a failure, or completely failed. + WaitingStatus WaitForThreadsRun_WaitingThreadStatus `protobuf:"varint,4,opt,name=waiting_status,json=waitingStatus,proto3,enum=littlehorse.WaitForThreadsRun_WaitingThreadStatus" json:"waiting_status,omitempty"` + // If there is a failure on the ThreadRun, and we have a failure handler defined + // for it, then we will start a failure handler for this threadrun. This field + // is the id of that threadRun. + FailureHandlerThreadRunId *int32 `protobuf:"varint,5,opt,name=failure_handler_thread_run_id,json=failureHandlerThreadRunId,proto3,oneof" json:"failure_handler_thread_run_id,omitempty"` } func (x *WaitForThreadsRun_WaitForThread) Reset() { @@ -928,11 +984,18 @@ func (x *WaitForThreadsRun_WaitForThread) GetThreadRunNumber() int32 { return 0 } -func (x *WaitForThreadsRun_WaitForThread) GetAlreadyHandled() bool { +func (x *WaitForThreadsRun_WaitForThread) GetWaitingStatus() WaitForThreadsRun_WaitingThreadStatus { if x != nil { - return x.AlreadyHandled + return x.WaitingStatus } - return false + return WaitForThreadsRun_THREAD_IN_PROGRESS +} + +func (x *WaitForThreadsRun_WaitForThread) GetFailureHandlerThreadRunId() int32 { + if x != nil && x.FailureHandlerThreadRunId != nil { + return *x.FailureHandlerThreadRunId + } + return 0 } var File_node_run_proto protoreflect.FileDescriptor @@ -1040,73 +1103,88 @@ var file_node_run_proto_rawDesc = []byte{ 0x69, 0x70, 0x6c, 0x65, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x52, 0x75, 0x6e, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, - 0x53, 0x70, 0x65, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x96, 0x03, 0x0a, 0x11, 0x57, 0x61, 0x69, + 0x53, 0x70, 0x65, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x85, 0x05, 0x0a, 0x11, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x52, 0x75, 0x6e, 0x12, 0x46, 0x0a, 0x07, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x52, 0x75, 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x52, 0x07, 0x74, - 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x12, 0x39, 0x0a, 0x06, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, - 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x54, 0x68, 0x72, 0x65, - 0x61, 0x64, 0x73, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x06, 0x70, 0x6f, 0x6c, 0x69, 0x63, - 0x79, 0x1a, 0xfd, 0x01, 0x0a, 0x0d, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x54, 0x68, 0x72, - 0x65, 0x61, 0x64, 0x12, 0x47, 0x0a, 0x0f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x65, 0x6e, - 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x00, 0x52, 0x0d, 0x74, 0x68, 0x72, 0x65, - 0x61, 0x64, 0x45, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x12, 0x3a, 0x0a, 0x0d, - 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, - 0x65, 0x2e, 0x4c, 0x48, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0c, 0x74, 0x68, 0x72, 0x65, - 0x61, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x74, 0x68, 0x72, 0x65, - 0x61, 0x64, 0x5f, 0x72, 0x75, 0x6e, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x0f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x52, 0x75, 0x6e, 0x4e, 0x75, - 0x6d, 0x62, 0x65, 0x72, 0x12, 0x27, 0x0a, 0x0f, 0x61, 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x5f, - 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x61, - 0x6c, 0x72, 0x65, 0x61, 0x64, 0x79, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x64, 0x42, 0x12, 0x0a, - 0x10, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, 0x6d, - 0x65, 0x22, 0x9a, 0x02, 0x0a, 0x10, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x52, 0x75, 0x6e, 0x12, 0x52, 0x0a, 0x15, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x64, 0x65, 0x66, 0x5f, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, - 0x72, 0x73, 0x65, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x44, 0x65, 0x66, 0x49, 0x64, 0x52, 0x12, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x66, 0x49, 0x64, 0x12, 0x3e, 0x0a, 0x0a, 0x65, 0x76, - 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x00, 0x52, 0x09, 0x65, 0x76, - 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x12, 0x4d, 0x0a, 0x11, 0x65, 0x78, - 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, - 0x72, 0x73, 0x65, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x49, 0x64, 0x48, 0x01, 0x52, 0x0f, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x65, 0x76, - 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x65, 0x78, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x22, 0x53, - 0x0a, 0x0c, 0x53, 0x6c, 0x65, 0x65, 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x75, 0x6e, 0x12, 0x43, - 0x0a, 0x0f, 0x6d, 0x61, 0x74, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x69, 0x6d, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x0e, 0x6d, 0x61, 0x74, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, - 0x69, 0x6d, 0x65, 0x22, 0xbf, 0x01, 0x0a, 0x07, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x12, - 0x21, 0x0a, 0x0c, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x39, 0x0a, 0x07, - 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, - 0x61, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x48, 0x00, 0x52, 0x07, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x88, 0x01, 0x01, 0x12, 0x30, 0x0a, 0x14, 0x77, 0x61, 0x73, 0x5f, 0x70, - 0x72, 0x6f, 0x70, 0x65, 0x72, 0x6c, 0x79, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x64, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x77, 0x61, 0x73, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, - 0x6c, 0x79, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x64, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x63, 0x6f, - 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x42, 0x47, 0x0a, 0x1f, 0x69, 0x6f, 0x2e, 0x6c, 0x69, 0x74, 0x74, - 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x73, 0x64, 0x6b, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, - 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x07, 0x2e, 0x3b, 0x6d, 0x6f, - 0x64, 0x65, 0x6c, 0xaa, 0x02, 0x18, 0x4c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x48, 0x6f, 0x72, 0x73, - 0x65, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x1a, 0x98, 0x03, 0x0a, 0x0d, 0x57, 0x61, 0x69, 0x74, 0x46, + 0x6f, 0x72, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x47, 0x0a, 0x0f, 0x74, 0x68, 0x72, 0x65, + 0x61, 0x64, 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x00, 0x52, + 0x0d, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x45, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x88, 0x01, + 0x01, 0x12, 0x3a, 0x0a, 0x0d, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x15, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, + 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x4c, 0x48, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x0c, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x2a, 0x0a, + 0x11, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x72, 0x75, 0x6e, 0x5f, 0x6e, 0x75, 0x6d, 0x62, + 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, + 0x52, 0x75, 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x59, 0x0a, 0x0e, 0x77, 0x61, 0x69, + 0x74, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x32, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, + 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x52, 0x75, + 0x6e, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0d, 0x77, 0x61, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x45, 0x0a, 0x1d, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, + 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x72, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x72, + 0x75, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x48, 0x01, 0x52, 0x19, 0x66, + 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x72, 0x54, 0x68, 0x72, + 0x65, 0x61, 0x64, 0x52, 0x75, 0x6e, 0x49, 0x64, 0x88, 0x01, 0x01, 0x42, 0x12, 0x0a, 0x10, 0x5f, + 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x65, 0x6e, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x42, + 0x20, 0x0a, 0x1e, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, 0x68, 0x61, 0x6e, 0x64, + 0x6c, 0x65, 0x72, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x72, 0x75, 0x6e, 0x5f, 0x69, + 0x64, 0x22, 0x8c, 0x01, 0x0a, 0x13, 0x57, 0x61, 0x69, 0x74, 0x69, 0x6e, 0x67, 0x54, 0x68, 0x72, + 0x65, 0x61, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x16, 0x0a, 0x12, 0x54, 0x48, 0x52, + 0x45, 0x41, 0x44, 0x5f, 0x49, 0x4e, 0x5f, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, 0x53, 0x53, 0x10, + 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x54, 0x48, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x48, 0x41, 0x4e, 0x44, + 0x4c, 0x49, 0x4e, 0x47, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x01, 0x12, 0x27, + 0x0a, 0x23, 0x54, 0x48, 0x52, 0x45, 0x41, 0x44, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, + 0x45, 0x44, 0x5f, 0x4f, 0x52, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x5f, 0x48, 0x41, + 0x4e, 0x44, 0x4c, 0x45, 0x44, 0x10, 0x02, 0x12, 0x17, 0x0a, 0x13, 0x54, 0x48, 0x52, 0x45, 0x41, + 0x44, 0x5f, 0x55, 0x4e, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x46, 0x55, 0x4c, 0x10, 0x03, + 0x22, 0x9a, 0x02, 0x0a, 0x10, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x76, 0x65, + 0x6e, 0x74, 0x52, 0x75, 0x6e, 0x12, 0x52, 0x0a, 0x15, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x64, 0x65, 0x66, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, + 0x73, 0x65, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x44, 0x65, 0x66, 0x49, 0x64, 0x52, 0x12, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x66, 0x49, 0x64, 0x12, 0x3e, 0x0a, 0x0a, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x00, 0x52, 0x09, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x88, 0x01, 0x01, 0x12, 0x4d, 0x0a, 0x11, 0x65, 0x78, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, + 0x73, 0x65, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x49, 0x64, 0x48, 0x01, 0x52, 0x0f, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x49, 0x64, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x42, 0x14, 0x0a, 0x12, 0x5f, 0x65, 0x78, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x22, 0x53, 0x0a, + 0x0c, 0x53, 0x6c, 0x65, 0x65, 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x75, 0x6e, 0x12, 0x43, 0x0a, + 0x0f, 0x6d, 0x61, 0x74, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x0e, 0x6d, 0x61, 0x74, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, + 0x6d, 0x65, 0x22, 0xbf, 0x01, 0x0a, 0x07, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x12, 0x21, + 0x0a, 0x0c, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x39, 0x0a, 0x07, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6c, + 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, + 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x48, 0x00, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x88, 0x01, 0x01, 0x12, 0x30, 0x0a, 0x14, 0x77, 0x61, 0x73, 0x5f, 0x70, 0x72, + 0x6f, 0x70, 0x65, 0x72, 0x6c, 0x79, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x64, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x77, 0x61, 0x73, 0x50, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x6c, + 0x79, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x64, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x42, 0x47, 0x0a, 0x1f, 0x69, 0x6f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, + 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x73, 0x64, 0x6b, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, + 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x07, 0x2e, 0x3b, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0xaa, 0x02, 0x18, 0x4c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x48, 0x6f, 0x72, 0x73, 0x65, + 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1121,58 +1199,59 @@ func file_node_run_proto_rawDescGZIP() []byte { return file_node_run_proto_rawDescData } +var file_node_run_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_node_run_proto_msgTypes = make([]protoimpl.MessageInfo, 12) var file_node_run_proto_goTypes = []interface{}{ - (*NodeRun)(nil), // 0: littlehorse.NodeRun - (*TaskNodeRun)(nil), // 1: littlehorse.TaskNodeRun - (*UserTaskNodeRun)(nil), // 2: littlehorse.UserTaskNodeRun - (*EntrypointRun)(nil), // 3: littlehorse.EntrypointRun - (*ExitRun)(nil), // 4: littlehorse.ExitRun - (*StartThreadRun)(nil), // 5: littlehorse.StartThreadRun - (*StartMultipleThreadsRun)(nil), // 6: littlehorse.StartMultipleThreadsRun - (*WaitForThreadsRun)(nil), // 7: littlehorse.WaitForThreadsRun - (*ExternalEventRun)(nil), // 8: littlehorse.ExternalEventRun - (*SleepNodeRun)(nil), // 9: littlehorse.SleepNodeRun - (*Failure)(nil), // 10: littlehorse.Failure - (*WaitForThreadsRun_WaitForThread)(nil), // 11: littlehorse.WaitForThreadsRun.WaitForThread - (*NodeRunId)(nil), // 12: littlehorse.NodeRunId - (*WfSpecId)(nil), // 13: littlehorse.WfSpecId - (LHStatus)(0), // 14: littlehorse.LHStatus - (*timestamppb.Timestamp)(nil), // 15: google.protobuf.Timestamp - (*TaskRunId)(nil), // 16: littlehorse.TaskRunId - (*UserTaskRunId)(nil), // 17: littlehorse.UserTaskRunId - (WaitForThreadsPolicy)(0), // 18: littlehorse.WaitForThreadsPolicy + (WaitForThreadsRun_WaitingThreadStatus)(0), // 0: littlehorse.WaitForThreadsRun.WaitingThreadStatus + (*NodeRun)(nil), // 1: littlehorse.NodeRun + (*TaskNodeRun)(nil), // 2: littlehorse.TaskNodeRun + (*UserTaskNodeRun)(nil), // 3: littlehorse.UserTaskNodeRun + (*EntrypointRun)(nil), // 4: littlehorse.EntrypointRun + (*ExitRun)(nil), // 5: littlehorse.ExitRun + (*StartThreadRun)(nil), // 6: littlehorse.StartThreadRun + (*StartMultipleThreadsRun)(nil), // 7: littlehorse.StartMultipleThreadsRun + (*WaitForThreadsRun)(nil), // 8: littlehorse.WaitForThreadsRun + (*ExternalEventRun)(nil), // 9: littlehorse.ExternalEventRun + (*SleepNodeRun)(nil), // 10: littlehorse.SleepNodeRun + (*Failure)(nil), // 11: littlehorse.Failure + (*WaitForThreadsRun_WaitForThread)(nil), // 12: littlehorse.WaitForThreadsRun.WaitForThread + (*NodeRunId)(nil), // 13: littlehorse.NodeRunId + (*WfSpecId)(nil), // 14: littlehorse.WfSpecId + (LHStatus)(0), // 15: littlehorse.LHStatus + (*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp + (*TaskRunId)(nil), // 17: littlehorse.TaskRunId + (*UserTaskRunId)(nil), // 18: littlehorse.UserTaskRunId (*ExternalEventDefId)(nil), // 19: littlehorse.ExternalEventDefId (*ExternalEventId)(nil), // 20: littlehorse.ExternalEventId (*VariableValue)(nil), // 21: littlehorse.VariableValue } var file_node_run_proto_depIdxs = []int32{ - 12, // 0: littlehorse.NodeRun.id:type_name -> littlehorse.NodeRunId - 13, // 1: littlehorse.NodeRun.wf_spec_id:type_name -> littlehorse.WfSpecId - 14, // 2: littlehorse.NodeRun.status:type_name -> littlehorse.LHStatus - 15, // 3: littlehorse.NodeRun.arrival_time:type_name -> google.protobuf.Timestamp - 15, // 4: littlehorse.NodeRun.end_time:type_name -> google.protobuf.Timestamp - 10, // 5: littlehorse.NodeRun.failures:type_name -> littlehorse.Failure - 1, // 6: littlehorse.NodeRun.task:type_name -> littlehorse.TaskNodeRun - 8, // 7: littlehorse.NodeRun.external_event:type_name -> littlehorse.ExternalEventRun - 3, // 8: littlehorse.NodeRun.entrypoint:type_name -> littlehorse.EntrypointRun - 4, // 9: littlehorse.NodeRun.exit:type_name -> littlehorse.ExitRun - 5, // 10: littlehorse.NodeRun.start_thread:type_name -> littlehorse.StartThreadRun - 7, // 11: littlehorse.NodeRun.wait_threads:type_name -> littlehorse.WaitForThreadsRun - 9, // 12: littlehorse.NodeRun.sleep:type_name -> littlehorse.SleepNodeRun - 2, // 13: littlehorse.NodeRun.user_task:type_name -> littlehorse.UserTaskNodeRun - 6, // 14: littlehorse.NodeRun.start_multiple_threads:type_name -> littlehorse.StartMultipleThreadsRun - 16, // 15: littlehorse.TaskNodeRun.task_run_id:type_name -> littlehorse.TaskRunId - 17, // 16: littlehorse.UserTaskNodeRun.user_task_run_id:type_name -> littlehorse.UserTaskRunId - 11, // 17: littlehorse.WaitForThreadsRun.threads:type_name -> littlehorse.WaitForThreadsRun.WaitForThread - 18, // 18: littlehorse.WaitForThreadsRun.policy:type_name -> littlehorse.WaitForThreadsPolicy - 19, // 19: littlehorse.ExternalEventRun.external_event_def_id:type_name -> littlehorse.ExternalEventDefId - 15, // 20: littlehorse.ExternalEventRun.event_time:type_name -> google.protobuf.Timestamp - 20, // 21: littlehorse.ExternalEventRun.external_event_id:type_name -> littlehorse.ExternalEventId - 15, // 22: littlehorse.SleepNodeRun.maturation_time:type_name -> google.protobuf.Timestamp - 21, // 23: littlehorse.Failure.content:type_name -> littlehorse.VariableValue - 15, // 24: littlehorse.WaitForThreadsRun.WaitForThread.thread_end_time:type_name -> google.protobuf.Timestamp - 14, // 25: littlehorse.WaitForThreadsRun.WaitForThread.thread_status:type_name -> littlehorse.LHStatus + 13, // 0: littlehorse.NodeRun.id:type_name -> littlehorse.NodeRunId + 14, // 1: littlehorse.NodeRun.wf_spec_id:type_name -> littlehorse.WfSpecId + 15, // 2: littlehorse.NodeRun.status:type_name -> littlehorse.LHStatus + 16, // 3: littlehorse.NodeRun.arrival_time:type_name -> google.protobuf.Timestamp + 16, // 4: littlehorse.NodeRun.end_time:type_name -> google.protobuf.Timestamp + 11, // 5: littlehorse.NodeRun.failures:type_name -> littlehorse.Failure + 2, // 6: littlehorse.NodeRun.task:type_name -> littlehorse.TaskNodeRun + 9, // 7: littlehorse.NodeRun.external_event:type_name -> littlehorse.ExternalEventRun + 4, // 8: littlehorse.NodeRun.entrypoint:type_name -> littlehorse.EntrypointRun + 5, // 9: littlehorse.NodeRun.exit:type_name -> littlehorse.ExitRun + 6, // 10: littlehorse.NodeRun.start_thread:type_name -> littlehorse.StartThreadRun + 8, // 11: littlehorse.NodeRun.wait_threads:type_name -> littlehorse.WaitForThreadsRun + 10, // 12: littlehorse.NodeRun.sleep:type_name -> littlehorse.SleepNodeRun + 3, // 13: littlehorse.NodeRun.user_task:type_name -> littlehorse.UserTaskNodeRun + 7, // 14: littlehorse.NodeRun.start_multiple_threads:type_name -> littlehorse.StartMultipleThreadsRun + 17, // 15: littlehorse.TaskNodeRun.task_run_id:type_name -> littlehorse.TaskRunId + 18, // 16: littlehorse.UserTaskNodeRun.user_task_run_id:type_name -> littlehorse.UserTaskRunId + 12, // 17: littlehorse.WaitForThreadsRun.threads:type_name -> littlehorse.WaitForThreadsRun.WaitForThread + 19, // 18: littlehorse.ExternalEventRun.external_event_def_id:type_name -> littlehorse.ExternalEventDefId + 16, // 19: littlehorse.ExternalEventRun.event_time:type_name -> google.protobuf.Timestamp + 20, // 20: littlehorse.ExternalEventRun.external_event_id:type_name -> littlehorse.ExternalEventId + 16, // 21: littlehorse.SleepNodeRun.maturation_time:type_name -> google.protobuf.Timestamp + 21, // 22: littlehorse.Failure.content:type_name -> littlehorse.VariableValue + 16, // 23: littlehorse.WaitForThreadsRun.WaitForThread.thread_end_time:type_name -> google.protobuf.Timestamp + 15, // 24: littlehorse.WaitForThreadsRun.WaitForThread.thread_status:type_name -> littlehorse.LHStatus + 0, // 25: littlehorse.WaitForThreadsRun.WaitForThread.waiting_status:type_name -> littlehorse.WaitForThreadsRun.WaitingThreadStatus 26, // [26:26] is the sub-list for method output_type 26, // [26:26] is the sub-list for method input_type 26, // [26:26] is the sub-list for extension type_name @@ -1356,13 +1435,14 @@ func file_node_run_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_node_run_proto_rawDesc, - NumEnums: 0, + NumEnums: 1, NumMessages: 12, NumExtensions: 0, NumServices: 0, }, GoTypes: file_node_run_proto_goTypes, DependencyIndexes: file_node_run_proto_depIdxs, + EnumInfos: file_node_run_proto_enumTypes, MessageInfos: file_node_run_proto_msgTypes, }.Build() File_node_run_proto = out.File diff --git a/sdk-go/common/model/wf_spec.pb.go b/sdk-go/common/model/wf_spec.pb.go index b77314905..be2ceb70a 100644 --- a/sdk-go/common/model/wf_spec.pb.go +++ b/sdk-go/common/model/wf_spec.pb.go @@ -885,11 +885,11 @@ type WaitForThreadsNode struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Either 1 or 3 is set. Cannot put `repeated` into a oneof, and - // for compatibility reasons, we cannot wrap it into a separate message. - Threads []*WaitForThreadsNode_ThreadToWaitFor `protobuf:"bytes,1,rep,name=threads,proto3" json:"threads,omitempty"` - ThreadList *VariableAssignment `protobuf:"bytes,3,opt,name=thread_list,json=threadList,proto3,oneof" json:"thread_list,omitempty"` - Policy WaitForThreadsPolicy `protobuf:"varint,2,opt,name=policy,proto3,enum=littlehorse.WaitForThreadsPolicy" json:"policy,omitempty"` + // Types that are assignable to ThreadsToWaitFor: + // *WaitForThreadsNode_Threads + // *WaitForThreadsNode_ThreadList + ThreadsToWaitFor isWaitForThreadsNode_ThreadsToWaitFor `protobuf_oneof:"threads_to_wait_for"` + PerThreadFailureHandlers []*FailureHandlerDef `protobuf:"bytes,3,rep,name=per_thread_failure_handlers,json=perThreadFailureHandlers,proto3" json:"per_thread_failure_handlers,omitempty"` } func (x *WaitForThreadsNode) Reset() { @@ -924,27 +924,50 @@ func (*WaitForThreadsNode) Descriptor() ([]byte, []int) { return file_wf_spec_proto_rawDescGZIP(), []int{11} } -func (x *WaitForThreadsNode) GetThreads() []*WaitForThreadsNode_ThreadToWaitFor { - if x != nil { +func (m *WaitForThreadsNode) GetThreadsToWaitFor() isWaitForThreadsNode_ThreadsToWaitFor { + if m != nil { + return m.ThreadsToWaitFor + } + return nil +} + +func (x *WaitForThreadsNode) GetThreads() *WaitForThreadsNode_ThreadsToWaitFor { + if x, ok := x.GetThreadsToWaitFor().(*WaitForThreadsNode_Threads); ok { return x.Threads } return nil } func (x *WaitForThreadsNode) GetThreadList() *VariableAssignment { - if x != nil { + if x, ok := x.GetThreadsToWaitFor().(*WaitForThreadsNode_ThreadList); ok { return x.ThreadList } return nil } -func (x *WaitForThreadsNode) GetPolicy() WaitForThreadsPolicy { +func (x *WaitForThreadsNode) GetPerThreadFailureHandlers() []*FailureHandlerDef { if x != nil { - return x.Policy + return x.PerThreadFailureHandlers } - return WaitForThreadsPolicy_STOP_ON_FAILURE + return nil +} + +type isWaitForThreadsNode_ThreadsToWaitFor interface { + isWaitForThreadsNode_ThreadsToWaitFor() } +type WaitForThreadsNode_Threads struct { + Threads *WaitForThreadsNode_ThreadsToWaitFor `protobuf:"bytes,1,opt,name=threads,proto3,oneof"` +} + +type WaitForThreadsNode_ThreadList struct { + ThreadList *VariableAssignment `protobuf:"bytes,2,opt,name=thread_list,json=threadList,proto3,oneof"` +} + +func (*WaitForThreadsNode_Threads) isWaitForThreadsNode_ThreadsToWaitFor() {} + +func (*WaitForThreadsNode_ThreadList) isWaitForThreadsNode_ThreadsToWaitFor() {} + type ExternalEventNode struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1983,6 +2006,53 @@ func (x *WaitForThreadsNode_ThreadToWaitFor) GetThreadRunNumber() *VariableAssig return nil } +type WaitForThreadsNode_ThreadsToWaitFor struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Threads []*WaitForThreadsNode_ThreadToWaitFor `protobuf:"bytes,1,rep,name=threads,proto3" json:"threads,omitempty"` +} + +func (x *WaitForThreadsNode_ThreadsToWaitFor) Reset() { + *x = WaitForThreadsNode_ThreadsToWaitFor{} + if protoimpl.UnsafeEnabled { + mi := &file_wf_spec_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *WaitForThreadsNode_ThreadsToWaitFor) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WaitForThreadsNode_ThreadsToWaitFor) ProtoMessage() {} + +func (x *WaitForThreadsNode_ThreadsToWaitFor) ProtoReflect() protoreflect.Message { + mi := &file_wf_spec_proto_msgTypes[31] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WaitForThreadsNode_ThreadsToWaitFor.ProtoReflect.Descriptor instead. +func (*WaitForThreadsNode_ThreadsToWaitFor) Descriptor() ([]byte, []int) { + return file_wf_spec_proto_rawDescGZIP(), []int{11, 1} +} + +func (x *WaitForThreadsNode_ThreadsToWaitFor) GetThreads() []*WaitForThreadsNode_ThreadToWaitFor { + if x != nil { + return x.Threads + } + return nil +} + var File_wf_spec_proto protoreflect.FileDescriptor var file_wf_spec_proto_rawDesc = []byte{ @@ -2174,219 +2244,228 @@ var file_wf_spec_proto_rawDesc = []byte{ 0x4f, 0x52, 0x10, 0x00, 0x12, 0x1a, 0x0a, 0x16, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x58, 0x43, 0x45, 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x10, 0x01, 0x42, 0x12, 0x0a, 0x10, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, 0x74, 0x6f, 0x5f, 0x63, - 0x61, 0x74, 0x63, 0x68, 0x22, 0xd1, 0x02, 0x0a, 0x12, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, - 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x49, 0x0a, 0x07, 0x74, - 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x6c, + 0x61, 0x74, 0x63, 0x68, 0x22, 0xdb, 0x03, 0x0a, 0x12, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, + 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x4c, 0x0a, 0x07, 0x74, + 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x4e, 0x6f, 0x64, 0x65, 0x2e, 0x54, 0x68, - 0x72, 0x65, 0x61, 0x64, 0x54, 0x6f, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x52, 0x07, 0x74, - 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x12, 0x45, 0x0a, 0x0b, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, - 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, - 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, - 0x6c, 0x65, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0a, - 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x4c, 0x69, 0x73, 0x74, 0x88, 0x01, 0x01, 0x12, 0x39, 0x0a, - 0x06, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, - 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x57, 0x61, 0x69, 0x74, - 0x46, 0x6f, 0x72, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, - 0x52, 0x06, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x1a, 0x5e, 0x0a, 0x0f, 0x54, 0x68, 0x72, 0x65, - 0x61, 0x64, 0x54, 0x6f, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x12, 0x4b, 0x0a, 0x11, 0x74, - 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x72, 0x75, 0x6e, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, - 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x73, 0x73, - 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x52, - 0x75, 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x74, 0x68, 0x72, - 0x65, 0x61, 0x64, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x22, 0xb1, 0x01, 0x0a, 0x11, 0x45, 0x78, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x52, - 0x0a, 0x15, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, - 0x5f, 0x64, 0x65, 0x66, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, - 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x45, 0x78, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x66, 0x49, 0x64, 0x52, 0x12, - 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x66, - 0x49, 0x64, 0x12, 0x48, 0x0a, 0x0f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, - 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, - 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, - 0x6c, 0x65, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0e, 0x74, 0x69, - 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x10, 0x0a, 0x0e, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x22, 0x59, - 0x0a, 0x08, 0x45, 0x78, 0x69, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x3d, 0x0a, 0x0b, 0x66, 0x61, - 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, 0x64, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x17, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x46, 0x61, - 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x66, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x61, 0x69, 0x6c, - 0x75, 0x72, 0x65, 0x44, 0x65, 0x66, 0x88, 0x01, 0x01, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x66, 0x61, - 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, 0x64, 0x65, 0x66, 0x22, 0x95, 0x01, 0x0a, 0x0a, 0x46, 0x61, - 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x66, 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x61, 0x69, 0x6c, - 0x75, 0x72, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x3e, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, - 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x73, 0x73, - 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x74, 0x88, 0x01, 0x01, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x22, 0xf8, 0x05, 0x0a, 0x04, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x38, 0x0a, 0x0e, 0x6f, 0x75, - 0x74, 0x67, 0x6f, 0x69, 0x6e, 0x67, 0x5f, 0x65, 0x64, 0x67, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, - 0x2e, 0x45, 0x64, 0x67, 0x65, 0x52, 0x0d, 0x6f, 0x75, 0x74, 0x67, 0x6f, 0x69, 0x6e, 0x67, 0x45, - 0x64, 0x67, 0x65, 0x73, 0x12, 0x49, 0x0a, 0x10, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, - 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, - 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x46, 0x61, 0x69, - 0x6c, 0x75, 0x72, 0x65, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x72, 0x44, 0x65, 0x66, 0x52, 0x0f, - 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x72, 0x73, 0x12, - 0x3d, 0x0a, 0x0a, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, - 0x65, 0x2e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x4e, 0x6f, 0x64, 0x65, - 0x48, 0x00, 0x52, 0x0a, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2b, - 0x0a, 0x04, 0x65, 0x78, 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6c, - 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x45, 0x78, 0x69, 0x74, 0x4e, - 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x04, 0x65, 0x78, 0x69, 0x74, 0x12, 0x2b, 0x0a, 0x04, 0x74, - 0x61, 0x73, 0x6b, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6c, 0x69, 0x74, 0x74, - 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x4e, 0x6f, 0x64, 0x65, - 0x48, 0x00, 0x52, 0x04, 0x74, 0x61, 0x73, 0x6b, 0x12, 0x47, 0x0a, 0x0e, 0x65, 0x78, 0x74, 0x65, - 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1e, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x45, - 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x64, 0x65, - 0x48, 0x00, 0x52, 0x0d, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x12, 0x41, 0x0a, 0x0c, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, - 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, - 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x68, 0x72, 0x65, 0x61, - 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x0b, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x68, - 0x72, 0x65, 0x61, 0x64, 0x12, 0x4b, 0x0a, 0x10, 0x77, 0x61, 0x69, 0x74, 0x5f, 0x66, 0x6f, 0x72, - 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, - 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x57, 0x61, 0x69, - 0x74, 0x46, 0x6f, 0x72, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x4e, 0x6f, 0x64, 0x65, 0x48, - 0x00, 0x52, 0x0e, 0x77, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, - 0x73, 0x12, 0x28, 0x0a, 0x03, 0x6e, 0x6f, 0x70, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, - 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x4e, 0x6f, 0x70, - 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x03, 0x6e, 0x6f, 0x70, 0x12, 0x2e, 0x0a, 0x05, 0x73, - 0x6c, 0x65, 0x65, 0x70, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6c, 0x69, 0x74, - 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x53, 0x6c, 0x65, 0x65, 0x70, 0x4e, 0x6f, - 0x64, 0x65, 0x48, 0x00, 0x52, 0x05, 0x73, 0x6c, 0x65, 0x65, 0x70, 0x12, 0x38, 0x0a, 0x09, 0x75, - 0x73, 0x65, 0x72, 0x5f, 0x74, 0x61, 0x73, 0x6b, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, - 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x55, 0x73, 0x65, - 0x72, 0x54, 0x61, 0x73, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x08, 0x75, 0x73, 0x65, - 0x72, 0x54, 0x61, 0x73, 0x6b, 0x12, 0x5d, 0x0a, 0x16, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x6d, - 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x18, - 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, - 0x72, 0x73, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, - 0x65, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x14, - 0x73, 0x74, 0x61, 0x72, 0x74, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x54, 0x68, 0x72, - 0x65, 0x61, 0x64, 0x73, 0x42, 0x06, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x22, 0xaa, 0x03, 0x0a, - 0x0c, 0x55, 0x73, 0x65, 0x72, 0x54, 0x61, 0x73, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x2b, 0x0a, - 0x12, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x64, 0x65, 0x66, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x75, 0x73, 0x65, 0x72, 0x54, - 0x61, 0x73, 0x6b, 0x44, 0x65, 0x66, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x43, 0x0a, 0x0a, 0x75, 0x73, - 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, + 0x72, 0x65, 0x61, 0x64, 0x73, 0x54, 0x6f, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x48, 0x00, + 0x52, 0x07, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x12, 0x42, 0x0a, 0x0b, 0x74, 0x68, 0x72, + 0x65, 0x61, 0x64, 0x5f, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, - 0x00, 0x52, 0x09, 0x75, 0x73, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x88, 0x01, 0x01, 0x12, - 0x3d, 0x0a, 0x07, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x00, 0x52, 0x0a, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x5d, 0x0a, + 0x1b, 0x70, 0x65, 0x72, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x66, 0x61, 0x69, 0x6c, + 0x75, 0x72, 0x65, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, + 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x72, 0x44, + 0x65, 0x66, 0x52, 0x18, 0x70, 0x65, 0x72, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x46, 0x61, 0x69, + 0x6c, 0x75, 0x72, 0x65, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x72, 0x73, 0x1a, 0x5e, 0x0a, 0x0f, + 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x54, 0x6f, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x12, + 0x4b, 0x0a, 0x11, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x72, 0x75, 0x6e, 0x5f, 0x6e, 0x75, + 0x6d, 0x62, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, + 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, + 0x65, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0f, 0x74, 0x68, 0x72, + 0x65, 0x61, 0x64, 0x52, 0x75, 0x6e, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x1a, 0x5d, 0x0a, 0x10, + 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x54, 0x6f, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, + 0x12, 0x49, 0x0a, 0x07, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x2f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, + 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x4e, 0x6f, + 0x64, 0x65, 0x2e, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x54, 0x6f, 0x57, 0x61, 0x69, 0x74, 0x46, + 0x6f, 0x72, 0x52, 0x07, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x42, 0x15, 0x0a, 0x13, 0x74, + 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x5f, 0x74, 0x6f, 0x5f, 0x77, 0x61, 0x69, 0x74, 0x5f, 0x66, + 0x6f, 0x72, 0x22, 0xb1, 0x01, 0x0a, 0x11, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x52, 0x0a, 0x15, 0x65, 0x78, 0x74, 0x65, + 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x64, 0x65, 0x66, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, + 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x44, 0x65, 0x66, 0x49, 0x64, 0x52, 0x12, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x44, 0x65, 0x66, 0x49, 0x64, 0x12, 0x48, 0x0a, 0x0f, + 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, + 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x73, 0x73, 0x69, + 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x53, + 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x10, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x70, + 0x6f, 0x69, 0x6e, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x22, 0x59, 0x0a, 0x08, 0x45, 0x78, 0x69, 0x74, + 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x3d, 0x0a, 0x0b, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, + 0x64, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6c, 0x69, 0x74, 0x74, + 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, + 0x65, 0x66, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x66, + 0x88, 0x01, 0x01, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, + 0x64, 0x65, 0x66, 0x22, 0x95, 0x01, 0x0a, 0x0a, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, + 0x65, 0x66, 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, + 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, + 0x3e, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, - 0x74, 0x48, 0x01, 0x52, 0x06, 0x75, 0x73, 0x65, 0x72, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x36, - 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x1c, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x55, 0x54, - 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x07, 0x61, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x36, 0x0a, 0x15, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, - 0x61, 0x73, 0x6b, 0x5f, 0x64, 0x65, 0x66, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x05, 0x48, 0x02, 0x52, 0x12, 0x75, 0x73, 0x65, 0x72, 0x54, 0x61, 0x73, - 0x6b, 0x44, 0x65, 0x66, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x3a, - 0x0a, 0x05, 0x6e, 0x6f, 0x74, 0x65, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, - 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, - 0x61, 0x62, 0x6c, 0x65, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x03, - 0x52, 0x05, 0x6e, 0x6f, 0x74, 0x65, 0x73, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x75, - 0x73, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x75, 0x73, - 0x65, 0x72, 0x5f, 0x69, 0x64, 0x42, 0x18, 0x0a, 0x16, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, - 0x61, 0x73, 0x6b, 0x5f, 0x64, 0x65, 0x66, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x42, - 0x08, 0x0a, 0x06, 0x5f, 0x6e, 0x6f, 0x74, 0x65, 0x73, 0x22, 0xb4, 0x01, 0x0a, 0x0d, 0x45, 0x64, - 0x67, 0x65, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x37, 0x0a, 0x0a, 0x63, - 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x17, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x43, 0x6f, - 0x6d, 0x70, 0x61, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x72, - 0x61, 0x74, 0x6f, 0x72, 0x12, 0x33, 0x0a, 0x04, 0x6c, 0x65, 0x66, 0x74, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, - 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, - 0x65, 0x6e, 0x74, 0x52, 0x04, 0x6c, 0x65, 0x66, 0x74, 0x12, 0x35, 0x0a, 0x05, 0x72, 0x69, 0x67, - 0x68, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, - 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x41, - 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x05, 0x72, 0x69, 0x67, 0x68, 0x74, - 0x22, 0xc7, 0x01, 0x0a, 0x04, 0x45, 0x64, 0x67, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x73, 0x69, 0x6e, - 0x6b, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0c, 0x73, 0x69, 0x6e, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, - 0x3d, 0x0a, 0x09, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, - 0x2e, 0x45, 0x64, 0x67, 0x65, 0x43, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, - 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x4c, - 0x0a, 0x12, 0x76, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6d, 0x75, 0x74, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6c, 0x69, 0x74, - 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, - 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x11, 0x76, 0x61, 0x72, 0x69, 0x61, - 0x62, 0x6c, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x0c, 0x0a, 0x0a, - 0x5f, 0x63, 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x09, 0x0a, 0x07, 0x4e, 0x6f, - 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x22, 0xde, 0x01, 0x0a, 0x09, 0x53, 0x6c, 0x65, 0x65, 0x70, 0x4e, - 0x6f, 0x64, 0x65, 0x12, 0x42, 0x0a, 0x0b, 0x72, 0x61, 0x77, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, - 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, + 0x74, 0x48, 0x00, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x88, 0x01, 0x01, 0x42, + 0x0a, 0x0a, 0x08, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x22, 0xf8, 0x05, 0x0a, 0x04, + 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x38, 0x0a, 0x0e, 0x6f, 0x75, 0x74, 0x67, 0x6f, 0x69, 0x6e, 0x67, + 0x5f, 0x65, 0x64, 0x67, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x6c, + 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x45, 0x64, 0x67, 0x65, 0x52, + 0x0d, 0x6f, 0x75, 0x74, 0x67, 0x6f, 0x69, 0x6e, 0x67, 0x45, 0x64, 0x67, 0x65, 0x73, 0x12, 0x49, + 0x0a, 0x10, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x6c, 0x65, + 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, + 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x48, 0x61, + 0x6e, 0x64, 0x6c, 0x65, 0x72, 0x44, 0x65, 0x66, 0x52, 0x0f, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, + 0x65, 0x48, 0x61, 0x6e, 0x64, 0x6c, 0x65, 0x72, 0x73, 0x12, 0x3d, 0x0a, 0x0a, 0x65, 0x6e, 0x74, + 0x72, 0x79, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, + 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x65, 0x6e, + 0x74, 0x72, 0x79, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2b, 0x0a, 0x04, 0x65, 0x78, 0x69, 0x74, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, + 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x45, 0x78, 0x69, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, + 0x04, 0x65, 0x78, 0x69, 0x74, 0x12, 0x2b, 0x0a, 0x04, 0x74, 0x61, 0x73, 0x6b, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, + 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x04, 0x74, 0x61, + 0x73, 0x6b, 0x12, 0x47, 0x0a, 0x0e, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6c, 0x69, 0x74, + 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x0d, 0x65, 0x78, + 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x41, 0x0a, 0x0c, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1c, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, + 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x4e, 0x6f, 0x64, 0x65, 0x48, + 0x00, 0x52, 0x0b, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x12, 0x4b, + 0x0a, 0x10, 0x77, 0x61, 0x69, 0x74, 0x5f, 0x66, 0x6f, 0x72, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, + 0x64, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, + 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, 0x72, 0x54, 0x68, + 0x72, 0x65, 0x61, 0x64, 0x73, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x0e, 0x77, 0x61, 0x69, + 0x74, 0x46, 0x6f, 0x72, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x12, 0x28, 0x0a, 0x03, 0x6e, + 0x6f, 0x70, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, + 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x4e, 0x6f, 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, + 0x52, 0x03, 0x6e, 0x6f, 0x70, 0x12, 0x2e, 0x0a, 0x05, 0x73, 0x6c, 0x65, 0x65, 0x70, 0x18, 0x0c, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, + 0x73, 0x65, 0x2e, 0x53, 0x6c, 0x65, 0x65, 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x05, + 0x73, 0x6c, 0x65, 0x65, 0x70, 0x12, 0x38, 0x0a, 0x09, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, 0x61, + 0x73, 0x6b, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, + 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x55, 0x73, 0x65, 0x72, 0x54, 0x61, 0x73, 0x6b, 0x4e, + 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x54, 0x61, 0x73, 0x6b, 0x12, + 0x5d, 0x0a, 0x16, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, + 0x65, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x25, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x53, 0x74, + 0x61, 0x72, 0x74, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x54, 0x68, 0x72, 0x65, 0x61, + 0x64, 0x73, 0x4e, 0x6f, 0x64, 0x65, 0x48, 0x00, 0x52, 0x14, 0x73, 0x74, 0x61, 0x72, 0x74, 0x4d, + 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x42, 0x06, + 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x22, 0xaa, 0x03, 0x0a, 0x0c, 0x55, 0x73, 0x65, 0x72, 0x54, + 0x61, 0x73, 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x2b, 0x0a, 0x12, 0x75, 0x73, 0x65, 0x72, 0x5f, + 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x64, 0x65, 0x66, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0f, 0x75, 0x73, 0x65, 0x72, 0x54, 0x61, 0x73, 0x6b, 0x44, 0x65, 0x66, + 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x43, 0x0a, 0x0a, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, + 0x75, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x41, - 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x72, 0x61, 0x77, - 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x3f, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, + 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x09, 0x75, 0x73, 0x65, + 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x88, 0x01, 0x01, 0x12, 0x3d, 0x0a, 0x07, 0x75, 0x73, 0x65, + 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, - 0x65, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x09, 0x74, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x3c, 0x0a, 0x08, 0x69, 0x73, 0x6f, 0x5f, - 0x64, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, - 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, - 0x65, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x07, 0x69, - 0x73, 0x6f, 0x44, 0x61, 0x74, 0x65, 0x42, 0x0e, 0x0a, 0x0c, 0x73, 0x6c, 0x65, 0x65, 0x70, 0x5f, - 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x22, 0xc7, 0x02, 0x0a, 0x16, 0x57, 0x66, 0x53, 0x70, 0x65, - 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x6e, 0x65, 0x77, 0x5f, 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x5f, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x6e, 0x65, - 0x77, 0x4d, 0x61, 0x6a, 0x6f, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, - 0x0c, 0x6e, 0x65, 0x77, 0x5f, 0x72, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x0b, 0x6e, 0x65, 0x77, 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x73, 0x0a, 0x16, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x5f, - 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x3d, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x57, - 0x66, 0x53, 0x70, 0x65, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4d, 0x69, 0x67, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x53, 0x70, 0x65, 0x63, - 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x14, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x53, 0x70, 0x65, 0x63, 0x4d, 0x69, 0x67, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x69, 0x0a, 0x19, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x53, - 0x70, 0x65, 0x63, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x36, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, - 0x65, 0x2e, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x53, 0x70, 0x65, 0x63, 0x4d, 0x69, 0x67, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0x84, 0x02, 0x0a, 0x13, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x53, 0x70, 0x65, 0x63, 0x4d, - 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2f, 0x0a, 0x14, 0x6e, 0x65, 0x77, 0x5f, - 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x5f, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x6e, 0x65, 0x77, 0x54, 0x68, 0x72, 0x65, 0x61, - 0x64, 0x53, 0x70, 0x65, 0x63, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x5d, 0x0a, 0x0f, 0x6e, 0x6f, 0x64, - 0x65, 0x5f, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, - 0x2e, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x53, 0x70, 0x65, 0x63, 0x4d, 0x69, 0x67, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0e, 0x6e, 0x6f, 0x64, 0x65, 0x4d, 0x69, - 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x5d, 0x0a, 0x13, 0x4e, 0x6f, 0x64, 0x65, - 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x30, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1a, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x4e, - 0x6f, 0x64, 0x65, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x33, 0x0a, 0x0d, 0x4e, 0x6f, 0x64, 0x65, 0x4d, - 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0d, 0x6e, 0x65, 0x77, 0x5f, - 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x6e, 0x65, 0x77, 0x4e, 0x6f, 0x64, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2a, 0x4e, 0x0a, 0x18, - 0x57, 0x66, 0x52, 0x75, 0x6e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x63, 0x63, - 0x65, 0x73, 0x73, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x0e, 0x0a, 0x0a, 0x50, 0x55, 0x42, 0x4c, - 0x49, 0x43, 0x5f, 0x56, 0x41, 0x52, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x52, 0x49, 0x56, - 0x41, 0x54, 0x45, 0x5f, 0x56, 0x41, 0x52, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x49, 0x4e, 0x48, - 0x45, 0x52, 0x49, 0x54, 0x45, 0x44, 0x5f, 0x56, 0x41, 0x52, 0x10, 0x02, 0x42, 0x47, 0x0a, 0x1f, - 0x69, 0x6f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x73, - 0x64, 0x6b, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x50, - 0x01, 0x5a, 0x07, 0x2e, 0x3b, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0xaa, 0x02, 0x18, 0x4c, 0x69, 0x74, - 0x74, 0x6c, 0x65, 0x48, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, - 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x01, 0x52, 0x06, 0x75, + 0x73, 0x65, 0x72, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x36, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x6c, 0x69, 0x74, 0x74, + 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x55, 0x54, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x54, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x12, 0x36, 0x0a, 0x15, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x64, 0x65, + 0x66, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x48, + 0x02, 0x52, 0x12, 0x75, 0x73, 0x65, 0x72, 0x54, 0x61, 0x73, 0x6b, 0x44, 0x65, 0x66, 0x56, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x3a, 0x0a, 0x05, 0x6e, 0x6f, 0x74, 0x65, + 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, + 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x73, + 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x03, 0x52, 0x05, 0x6e, 0x6f, 0x74, 0x65, + 0x73, 0x88, 0x01, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x67, 0x72, + 0x6f, 0x75, 0x70, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x42, + 0x18, 0x0a, 0x16, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x5f, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x64, 0x65, + 0x66, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x6e, 0x6f, + 0x74, 0x65, 0x73, 0x22, 0xb4, 0x01, 0x0a, 0x0d, 0x45, 0x64, 0x67, 0x65, 0x43, 0x6f, 0x6e, 0x64, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x37, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x61, + 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x6c, 0x69, 0x74, 0x74, + 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x61, 0x74, + 0x6f, 0x72, 0x52, 0x0a, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x33, + 0x0a, 0x04, 0x6c, 0x65, 0x66, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, + 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, + 0x62, 0x6c, 0x65, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x04, 0x6c, + 0x65, 0x66, 0x74, 0x12, 0x35, 0x0a, 0x05, 0x72, 0x69, 0x67, 0x68, 0x74, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, + 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x52, 0x05, 0x72, 0x69, 0x67, 0x68, 0x74, 0x22, 0xc7, 0x01, 0x0a, 0x04, 0x45, + 0x64, 0x67, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x73, 0x69, 0x6e, 0x6b, 0x5f, 0x6e, 0x6f, 0x64, 0x65, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, 0x69, 0x6e, + 0x6b, 0x4e, 0x6f, 0x64, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x3d, 0x0a, 0x09, 0x63, 0x6f, 0x6e, + 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6c, + 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x45, 0x64, 0x67, 0x65, 0x43, + 0x6f, 0x6e, 0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x09, 0x63, 0x6f, 0x6e, 0x64, + 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x88, 0x01, 0x01, 0x12, 0x4c, 0x0a, 0x12, 0x76, 0x61, 0x72, 0x69, + 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, + 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x11, 0x76, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x4d, 0x75, 0x74, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x63, 0x6f, 0x6e, 0x64, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x09, 0x0a, 0x07, 0x4e, 0x6f, 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x22, + 0xde, 0x01, 0x0a, 0x09, 0x53, 0x6c, 0x65, 0x65, 0x70, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x42, 0x0a, + 0x0b, 0x72, 0x61, 0x77, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, + 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x72, 0x61, 0x77, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, + 0x73, 0x12, 0x3f, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, + 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x73, 0x73, 0x69, 0x67, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x12, 0x3c, 0x0a, 0x08, 0x69, 0x73, 0x6f, 0x5f, 0x64, 0x61, 0x74, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, + 0x73, 0x65, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x73, 0x73, 0x69, 0x67, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x07, 0x69, 0x73, 0x6f, 0x44, 0x61, 0x74, 0x65, + 0x42, 0x0e, 0x0a, 0x0c, 0x73, 0x6c, 0x65, 0x65, 0x70, 0x5f, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, + 0x22, 0xc7, 0x02, 0x0a, 0x16, 0x57, 0x66, 0x53, 0x70, 0x65, 0x63, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x11, 0x6e, + 0x65, 0x77, 0x5f, 0x6d, 0x61, 0x6a, 0x6f, 0x72, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x6e, 0x65, 0x77, 0x4d, 0x61, 0x6a, 0x6f, 0x72, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x6e, 0x65, 0x77, 0x5f, 0x72, + 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x6e, + 0x65, 0x77, 0x52, 0x65, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x73, 0x0a, 0x16, 0x74, 0x68, + 0x72, 0x65, 0x61, 0x64, 0x5f, 0x73, 0x70, 0x65, 0x63, 0x5f, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x6c, 0x69, 0x74, + 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x57, 0x66, 0x53, 0x70, 0x65, 0x63, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x53, 0x70, 0x65, 0x63, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x14, 0x74, 0x68, 0x72, 0x65, 0x61, + 0x64, 0x53, 0x70, 0x65, 0x63, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, + 0x69, 0x0a, 0x19, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x53, 0x70, 0x65, 0x63, 0x4d, 0x69, 0x67, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x36, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, + 0x6c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x54, 0x68, 0x72, 0x65, + 0x61, 0x64, 0x53, 0x70, 0x65, 0x63, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x84, 0x02, 0x0a, 0x13, 0x54, + 0x68, 0x72, 0x65, 0x61, 0x64, 0x53, 0x70, 0x65, 0x63, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x2f, 0x0a, 0x14, 0x6e, 0x65, 0x77, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, + 0x5f, 0x73, 0x70, 0x65, 0x63, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x11, 0x6e, 0x65, 0x77, 0x54, 0x68, 0x72, 0x65, 0x61, 0x64, 0x53, 0x70, 0x65, 0x63, 0x4e, + 0x61, 0x6d, 0x65, 0x12, 0x5d, 0x0a, 0x0f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6d, 0x69, 0x67, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x6c, + 0x69, 0x74, 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x54, 0x68, 0x72, 0x65, 0x61, + 0x64, 0x53, 0x70, 0x65, 0x63, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x4e, + 0x6f, 0x64, 0x65, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x0e, 0x6e, 0x6f, 0x64, 0x65, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x1a, 0x5d, 0x0a, 0x13, 0x4e, 0x6f, 0x64, 0x65, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x30, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6c, 0x69, 0x74, + 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x4d, 0x69, 0x67, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x22, 0x33, 0x0a, 0x0d, 0x4e, 0x6f, 0x64, 0x65, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0d, 0x6e, 0x65, 0x77, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6e, 0x65, 0x77, 0x4e, 0x6f, + 0x64, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x2a, 0x4e, 0x0a, 0x18, 0x57, 0x66, 0x52, 0x75, 0x6e, 0x56, + 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4c, 0x65, 0x76, + 0x65, 0x6c, 0x12, 0x0e, 0x0a, 0x0a, 0x50, 0x55, 0x42, 0x4c, 0x49, 0x43, 0x5f, 0x56, 0x41, 0x52, + 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x52, 0x49, 0x56, 0x41, 0x54, 0x45, 0x5f, 0x56, 0x41, + 0x52, 0x10, 0x01, 0x12, 0x11, 0x0a, 0x0d, 0x49, 0x4e, 0x48, 0x45, 0x52, 0x49, 0x54, 0x45, 0x44, + 0x5f, 0x56, 0x41, 0x52, 0x10, 0x02, 0x42, 0x47, 0x0a, 0x1f, 0x69, 0x6f, 0x2e, 0x6c, 0x69, 0x74, + 0x74, 0x6c, 0x65, 0x68, 0x6f, 0x72, 0x73, 0x65, 0x2e, 0x73, 0x64, 0x6b, 0x2e, 0x63, 0x6f, 0x6d, + 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x07, 0x2e, 0x3b, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0xaa, 0x02, 0x18, 0x4c, 0x69, 0x74, 0x74, 0x6c, 0x65, 0x48, 0x6f, 0x72, + 0x73, 0x65, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -2402,86 +2481,86 @@ func file_wf_spec_proto_rawDescGZIP() []byte { } var file_wf_spec_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_wf_spec_proto_msgTypes = make([]protoimpl.MessageInfo, 33) +var file_wf_spec_proto_msgTypes = make([]protoimpl.MessageInfo, 34) var file_wf_spec_proto_goTypes = []interface{}{ - (WfRunVariableAccessLevel)(0), // 0: littlehorse.WfRunVariableAccessLevel - (FailureHandlerDef_LHFailureType)(0), // 1: littlehorse.FailureHandlerDef.LHFailureType - (*WfSpec)(nil), // 2: littlehorse.WfSpec - (*WorkflowRetentionPolicy)(nil), // 3: littlehorse.WorkflowRetentionPolicy - (*JsonIndex)(nil), // 4: littlehorse.JsonIndex - (*SearchableVariableDef)(nil), // 5: littlehorse.SearchableVariableDef - (*ThreadVarDef)(nil), // 6: littlehorse.ThreadVarDef - (*ThreadSpec)(nil), // 7: littlehorse.ThreadSpec - (*ThreadRetentionPolicy)(nil), // 8: littlehorse.ThreadRetentionPolicy - (*InterruptDef)(nil), // 9: littlehorse.InterruptDef - (*StartThreadNode)(nil), // 10: littlehorse.StartThreadNode - (*StartMultipleThreadsNode)(nil), // 11: littlehorse.StartMultipleThreadsNode - (*FailureHandlerDef)(nil), // 12: littlehorse.FailureHandlerDef - (*WaitForThreadsNode)(nil), // 13: littlehorse.WaitForThreadsNode - (*ExternalEventNode)(nil), // 14: littlehorse.ExternalEventNode - (*EntrypointNode)(nil), // 15: littlehorse.EntrypointNode - (*ExitNode)(nil), // 16: littlehorse.ExitNode - (*FailureDef)(nil), // 17: littlehorse.FailureDef - (*Node)(nil), // 18: littlehorse.Node - (*UserTaskNode)(nil), // 19: littlehorse.UserTaskNode - (*EdgeCondition)(nil), // 20: littlehorse.EdgeCondition - (*Edge)(nil), // 21: littlehorse.Edge - (*NopNode)(nil), // 22: littlehorse.NopNode - (*SleepNode)(nil), // 23: littlehorse.SleepNode - (*WfSpecVersionMigration)(nil), // 24: littlehorse.WfSpecVersionMigration - (*ThreadSpecMigration)(nil), // 25: littlehorse.ThreadSpecMigration - (*NodeMigration)(nil), // 26: littlehorse.NodeMigration - nil, // 27: littlehorse.WfSpec.ThreadSpecsEntry - (*WfSpec_ParentWfSpecReference)(nil), // 28: littlehorse.WfSpec.ParentWfSpecReference - nil, // 29: littlehorse.ThreadSpec.NodesEntry - nil, // 30: littlehorse.StartThreadNode.VariablesEntry - nil, // 31: littlehorse.StartMultipleThreadsNode.VariablesEntry - (*WaitForThreadsNode_ThreadToWaitFor)(nil), // 32: littlehorse.WaitForThreadsNode.ThreadToWaitFor - nil, // 33: littlehorse.WfSpecVersionMigration.ThreadSpecMigrationsEntry - nil, // 34: littlehorse.ThreadSpecMigration.NodeMigrationsEntry - (*WfSpecId)(nil), // 35: littlehorse.WfSpecId - (*timestamppb.Timestamp)(nil), // 36: google.protobuf.Timestamp - (MetadataStatus)(0), // 37: littlehorse.MetadataStatus - (VariableType)(0), // 38: littlehorse.VariableType - (*VariableDef)(nil), // 39: littlehorse.VariableDef - (*ExternalEventDefId)(nil), // 40: littlehorse.ExternalEventDefId - (*VariableAssignment)(nil), // 41: littlehorse.VariableAssignment - (WaitForThreadsPolicy)(0), // 42: littlehorse.WaitForThreadsPolicy + (WfRunVariableAccessLevel)(0), // 0: littlehorse.WfRunVariableAccessLevel + (FailureHandlerDef_LHFailureType)(0), // 1: littlehorse.FailureHandlerDef.LHFailureType + (*WfSpec)(nil), // 2: littlehorse.WfSpec + (*WorkflowRetentionPolicy)(nil), // 3: littlehorse.WorkflowRetentionPolicy + (*JsonIndex)(nil), // 4: littlehorse.JsonIndex + (*SearchableVariableDef)(nil), // 5: littlehorse.SearchableVariableDef + (*ThreadVarDef)(nil), // 6: littlehorse.ThreadVarDef + (*ThreadSpec)(nil), // 7: littlehorse.ThreadSpec + (*ThreadRetentionPolicy)(nil), // 8: littlehorse.ThreadRetentionPolicy + (*InterruptDef)(nil), // 9: littlehorse.InterruptDef + (*StartThreadNode)(nil), // 10: littlehorse.StartThreadNode + (*StartMultipleThreadsNode)(nil), // 11: littlehorse.StartMultipleThreadsNode + (*FailureHandlerDef)(nil), // 12: littlehorse.FailureHandlerDef + (*WaitForThreadsNode)(nil), // 13: littlehorse.WaitForThreadsNode + (*ExternalEventNode)(nil), // 14: littlehorse.ExternalEventNode + (*EntrypointNode)(nil), // 15: littlehorse.EntrypointNode + (*ExitNode)(nil), // 16: littlehorse.ExitNode + (*FailureDef)(nil), // 17: littlehorse.FailureDef + (*Node)(nil), // 18: littlehorse.Node + (*UserTaskNode)(nil), // 19: littlehorse.UserTaskNode + (*EdgeCondition)(nil), // 20: littlehorse.EdgeCondition + (*Edge)(nil), // 21: littlehorse.Edge + (*NopNode)(nil), // 22: littlehorse.NopNode + (*SleepNode)(nil), // 23: littlehorse.SleepNode + (*WfSpecVersionMigration)(nil), // 24: littlehorse.WfSpecVersionMigration + (*ThreadSpecMigration)(nil), // 25: littlehorse.ThreadSpecMigration + (*NodeMigration)(nil), // 26: littlehorse.NodeMigration + nil, // 27: littlehorse.WfSpec.ThreadSpecsEntry + (*WfSpec_ParentWfSpecReference)(nil), // 28: littlehorse.WfSpec.ParentWfSpecReference + nil, // 29: littlehorse.ThreadSpec.NodesEntry + nil, // 30: littlehorse.StartThreadNode.VariablesEntry + nil, // 31: littlehorse.StartMultipleThreadsNode.VariablesEntry + (*WaitForThreadsNode_ThreadToWaitFor)(nil), // 32: littlehorse.WaitForThreadsNode.ThreadToWaitFor + (*WaitForThreadsNode_ThreadsToWaitFor)(nil), // 33: littlehorse.WaitForThreadsNode.ThreadsToWaitFor + nil, // 34: littlehorse.WfSpecVersionMigration.ThreadSpecMigrationsEntry + nil, // 35: littlehorse.ThreadSpecMigration.NodeMigrationsEntry + (*WfSpecId)(nil), // 36: littlehorse.WfSpecId + (*timestamppb.Timestamp)(nil), // 37: google.protobuf.Timestamp + (MetadataStatus)(0), // 38: littlehorse.MetadataStatus + (VariableType)(0), // 39: littlehorse.VariableType + (*VariableDef)(nil), // 40: littlehorse.VariableDef + (*ExternalEventDefId)(nil), // 41: littlehorse.ExternalEventDefId + (*VariableAssignment)(nil), // 42: littlehorse.VariableAssignment (*TaskNode)(nil), // 43: littlehorse.TaskNode (*UTActionTrigger)(nil), // 44: littlehorse.UTActionTrigger (Comparator)(0), // 45: littlehorse.Comparator (*VariableMutation)(nil), // 46: littlehorse.VariableMutation } var file_wf_spec_proto_depIdxs = []int32{ - 35, // 0: littlehorse.WfSpec.id:type_name -> littlehorse.WfSpecId - 36, // 1: littlehorse.WfSpec.created_at:type_name -> google.protobuf.Timestamp + 36, // 0: littlehorse.WfSpec.id:type_name -> littlehorse.WfSpecId + 37, // 1: littlehorse.WfSpec.created_at:type_name -> google.protobuf.Timestamp 6, // 2: littlehorse.WfSpec.frozen_variables:type_name -> littlehorse.ThreadVarDef - 37, // 3: littlehorse.WfSpec.status:type_name -> littlehorse.MetadataStatus + 38, // 3: littlehorse.WfSpec.status:type_name -> littlehorse.MetadataStatus 27, // 4: littlehorse.WfSpec.thread_specs:type_name -> littlehorse.WfSpec.ThreadSpecsEntry 3, // 5: littlehorse.WfSpec.retention_policy:type_name -> littlehorse.WorkflowRetentionPolicy 24, // 6: littlehorse.WfSpec.migration:type_name -> littlehorse.WfSpecVersionMigration 28, // 7: littlehorse.WfSpec.parent_wf_spec:type_name -> littlehorse.WfSpec.ParentWfSpecReference - 38, // 8: littlehorse.JsonIndex.field_type:type_name -> littlehorse.VariableType - 39, // 9: littlehorse.SearchableVariableDef.var_def:type_name -> littlehorse.VariableDef - 39, // 10: littlehorse.ThreadVarDef.var_def:type_name -> littlehorse.VariableDef + 39, // 8: littlehorse.JsonIndex.field_type:type_name -> littlehorse.VariableType + 40, // 9: littlehorse.SearchableVariableDef.var_def:type_name -> littlehorse.VariableDef + 40, // 10: littlehorse.ThreadVarDef.var_def:type_name -> littlehorse.VariableDef 4, // 11: littlehorse.ThreadVarDef.json_indexes:type_name -> littlehorse.JsonIndex 0, // 12: littlehorse.ThreadVarDef.access_level:type_name -> littlehorse.WfRunVariableAccessLevel 29, // 13: littlehorse.ThreadSpec.nodes:type_name -> littlehorse.ThreadSpec.NodesEntry 6, // 14: littlehorse.ThreadSpec.variable_defs:type_name -> littlehorse.ThreadVarDef 9, // 15: littlehorse.ThreadSpec.interrupt_defs:type_name -> littlehorse.InterruptDef 8, // 16: littlehorse.ThreadSpec.retention_policy:type_name -> littlehorse.ThreadRetentionPolicy - 40, // 17: littlehorse.InterruptDef.external_event_def_id:type_name -> littlehorse.ExternalEventDefId + 41, // 17: littlehorse.InterruptDef.external_event_def_id:type_name -> littlehorse.ExternalEventDefId 30, // 18: littlehorse.StartThreadNode.variables:type_name -> littlehorse.StartThreadNode.VariablesEntry 31, // 19: littlehorse.StartMultipleThreadsNode.variables:type_name -> littlehorse.StartMultipleThreadsNode.VariablesEntry - 41, // 20: littlehorse.StartMultipleThreadsNode.iterable:type_name -> littlehorse.VariableAssignment + 42, // 20: littlehorse.StartMultipleThreadsNode.iterable:type_name -> littlehorse.VariableAssignment 1, // 21: littlehorse.FailureHandlerDef.any_failure_of_type:type_name -> littlehorse.FailureHandlerDef.LHFailureType - 32, // 22: littlehorse.WaitForThreadsNode.threads:type_name -> littlehorse.WaitForThreadsNode.ThreadToWaitFor - 41, // 23: littlehorse.WaitForThreadsNode.thread_list:type_name -> littlehorse.VariableAssignment - 42, // 24: littlehorse.WaitForThreadsNode.policy:type_name -> littlehorse.WaitForThreadsPolicy - 40, // 25: littlehorse.ExternalEventNode.external_event_def_id:type_name -> littlehorse.ExternalEventDefId - 41, // 26: littlehorse.ExternalEventNode.timeout_seconds:type_name -> littlehorse.VariableAssignment + 33, // 22: littlehorse.WaitForThreadsNode.threads:type_name -> littlehorse.WaitForThreadsNode.ThreadsToWaitFor + 42, // 23: littlehorse.WaitForThreadsNode.thread_list:type_name -> littlehorse.VariableAssignment + 12, // 24: littlehorse.WaitForThreadsNode.per_thread_failure_handlers:type_name -> littlehorse.FailureHandlerDef + 41, // 25: littlehorse.ExternalEventNode.external_event_def_id:type_name -> littlehorse.ExternalEventDefId + 42, // 26: littlehorse.ExternalEventNode.timeout_seconds:type_name -> littlehorse.VariableAssignment 17, // 27: littlehorse.ExitNode.failure_def:type_name -> littlehorse.FailureDef - 41, // 28: littlehorse.FailureDef.content:type_name -> littlehorse.VariableAssignment + 42, // 28: littlehorse.FailureDef.content:type_name -> littlehorse.VariableAssignment 21, // 29: littlehorse.Node.outgoing_edges:type_name -> littlehorse.Edge 12, // 30: littlehorse.Node.failure_handlers:type_name -> littlehorse.FailureHandlerDef 15, // 31: littlehorse.Node.entrypoint:type_name -> littlehorse.EntrypointNode @@ -2494,32 +2573,33 @@ var file_wf_spec_proto_depIdxs = []int32{ 23, // 38: littlehorse.Node.sleep:type_name -> littlehorse.SleepNode 19, // 39: littlehorse.Node.user_task:type_name -> littlehorse.UserTaskNode 11, // 40: littlehorse.Node.start_multiple_threads:type_name -> littlehorse.StartMultipleThreadsNode - 41, // 41: littlehorse.UserTaskNode.user_group:type_name -> littlehorse.VariableAssignment - 41, // 42: littlehorse.UserTaskNode.user_id:type_name -> littlehorse.VariableAssignment + 42, // 41: littlehorse.UserTaskNode.user_group:type_name -> littlehorse.VariableAssignment + 42, // 42: littlehorse.UserTaskNode.user_id:type_name -> littlehorse.VariableAssignment 44, // 43: littlehorse.UserTaskNode.actions:type_name -> littlehorse.UTActionTrigger - 41, // 44: littlehorse.UserTaskNode.notes:type_name -> littlehorse.VariableAssignment + 42, // 44: littlehorse.UserTaskNode.notes:type_name -> littlehorse.VariableAssignment 45, // 45: littlehorse.EdgeCondition.comparator:type_name -> littlehorse.Comparator - 41, // 46: littlehorse.EdgeCondition.left:type_name -> littlehorse.VariableAssignment - 41, // 47: littlehorse.EdgeCondition.right:type_name -> littlehorse.VariableAssignment + 42, // 46: littlehorse.EdgeCondition.left:type_name -> littlehorse.VariableAssignment + 42, // 47: littlehorse.EdgeCondition.right:type_name -> littlehorse.VariableAssignment 20, // 48: littlehorse.Edge.condition:type_name -> littlehorse.EdgeCondition 46, // 49: littlehorse.Edge.variable_mutations:type_name -> littlehorse.VariableMutation - 41, // 50: littlehorse.SleepNode.raw_seconds:type_name -> littlehorse.VariableAssignment - 41, // 51: littlehorse.SleepNode.timestamp:type_name -> littlehorse.VariableAssignment - 41, // 52: littlehorse.SleepNode.iso_date:type_name -> littlehorse.VariableAssignment - 33, // 53: littlehorse.WfSpecVersionMigration.thread_spec_migrations:type_name -> littlehorse.WfSpecVersionMigration.ThreadSpecMigrationsEntry - 34, // 54: littlehorse.ThreadSpecMigration.node_migrations:type_name -> littlehorse.ThreadSpecMigration.NodeMigrationsEntry + 42, // 50: littlehorse.SleepNode.raw_seconds:type_name -> littlehorse.VariableAssignment + 42, // 51: littlehorse.SleepNode.timestamp:type_name -> littlehorse.VariableAssignment + 42, // 52: littlehorse.SleepNode.iso_date:type_name -> littlehorse.VariableAssignment + 34, // 53: littlehorse.WfSpecVersionMigration.thread_spec_migrations:type_name -> littlehorse.WfSpecVersionMigration.ThreadSpecMigrationsEntry + 35, // 54: littlehorse.ThreadSpecMigration.node_migrations:type_name -> littlehorse.ThreadSpecMigration.NodeMigrationsEntry 7, // 55: littlehorse.WfSpec.ThreadSpecsEntry.value:type_name -> littlehorse.ThreadSpec 18, // 56: littlehorse.ThreadSpec.NodesEntry.value:type_name -> littlehorse.Node - 41, // 57: littlehorse.StartThreadNode.VariablesEntry.value:type_name -> littlehorse.VariableAssignment - 41, // 58: littlehorse.StartMultipleThreadsNode.VariablesEntry.value:type_name -> littlehorse.VariableAssignment - 41, // 59: littlehorse.WaitForThreadsNode.ThreadToWaitFor.thread_run_number:type_name -> littlehorse.VariableAssignment - 25, // 60: littlehorse.WfSpecVersionMigration.ThreadSpecMigrationsEntry.value:type_name -> littlehorse.ThreadSpecMigration - 26, // 61: littlehorse.ThreadSpecMigration.NodeMigrationsEntry.value:type_name -> littlehorse.NodeMigration - 62, // [62:62] is the sub-list for method output_type - 62, // [62:62] is the sub-list for method input_type - 62, // [62:62] is the sub-list for extension type_name - 62, // [62:62] is the sub-list for extension extendee - 0, // [0:62] is the sub-list for field type_name + 42, // 57: littlehorse.StartThreadNode.VariablesEntry.value:type_name -> littlehorse.VariableAssignment + 42, // 58: littlehorse.StartMultipleThreadsNode.VariablesEntry.value:type_name -> littlehorse.VariableAssignment + 42, // 59: littlehorse.WaitForThreadsNode.ThreadToWaitFor.thread_run_number:type_name -> littlehorse.VariableAssignment + 32, // 60: littlehorse.WaitForThreadsNode.ThreadsToWaitFor.threads:type_name -> littlehorse.WaitForThreadsNode.ThreadToWaitFor + 25, // 61: littlehorse.WfSpecVersionMigration.ThreadSpecMigrationsEntry.value:type_name -> littlehorse.ThreadSpecMigration + 26, // 62: littlehorse.ThreadSpecMigration.NodeMigrationsEntry.value:type_name -> littlehorse.NodeMigration + 63, // [63:63] is the sub-list for method output_type + 63, // [63:63] is the sub-list for method input_type + 63, // [63:63] is the sub-list for extension type_name + 63, // [63:63] is the sub-list for extension extendee + 0, // [0:63] is the sub-list for field type_name } func init() { file_wf_spec_proto_init() } @@ -2855,6 +2935,18 @@ func file_wf_spec_proto_init() { return nil } } + file_wf_spec_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*WaitForThreadsNode_ThreadsToWaitFor); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } file_wf_spec_proto_msgTypes[0].OneofWrappers = []interface{}{} file_wf_spec_proto_msgTypes[1].OneofWrappers = []interface{}{ @@ -2868,7 +2960,10 @@ func file_wf_spec_proto_init() { (*FailureHandlerDef_SpecificFailure)(nil), (*FailureHandlerDef_AnyFailureOfType)(nil), } - file_wf_spec_proto_msgTypes[11].OneofWrappers = []interface{}{} + file_wf_spec_proto_msgTypes[11].OneofWrappers = []interface{}{ + (*WaitForThreadsNode_Threads)(nil), + (*WaitForThreadsNode_ThreadList)(nil), + } file_wf_spec_proto_msgTypes[14].OneofWrappers = []interface{}{} file_wf_spec_proto_msgTypes[15].OneofWrappers = []interface{}{} file_wf_spec_proto_msgTypes[16].OneofWrappers = []interface{}{ @@ -2896,7 +2991,7 @@ func file_wf_spec_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_wf_spec_proto_rawDesc, NumEnums: 2, - NumMessages: 33, + NumMessages: 34, NumExtensions: 0, NumServices: 0, }, diff --git a/sdk-go/wflib/wf_lib_internal.go b/sdk-go/wflib/wf_lib_internal.go index 58cfe66ad..2a095b3c7 100644 --- a/sdk-go/wflib/wf_lib_internal.go +++ b/sdk-go/wflib/wf_lib_internal.go @@ -789,14 +789,18 @@ func (t *WorkflowThread) waitForThreads(s ...*SpawnedThread) *NodeOutput { nodeName, node := t.createBlankNode("wait", "WAIT_THREADS") node.Node = &model.Node_WaitForThreads{ WaitForThreads: &model.WaitForThreadsNode{ - Threads: make([]*model.WaitForThreadsNode_ThreadToWaitFor, 0), + ThreadsToWaitFor: &model.WaitForThreadsNode_Threads{ + Threads: &model.WaitForThreadsNode_ThreadsToWaitFor{ + Threads: make([]*model.WaitForThreadsNode_ThreadToWaitFor, 0), + }, + }, }, } for _, spawnedThread := range s { threadRunNumberAssn, _ := t.assignVariable(spawnedThread.threadNumVar) - node.GetWaitForThreads().Threads = append(node.GetWaitForThreads().Threads, + node.GetWaitForThreads().GetThreads().Threads = append(node.GetWaitForThreads().GetThreads().Threads, &model.WaitForThreadsNode_ThreadToWaitFor{ ThreadRunNumber: threadRunNumberAssn, }, @@ -870,8 +874,9 @@ func (t *WorkflowThread) waitForThreadsList(s *SpawnedThreads) NodeOutput { } subNode := &model.WaitForThreadsNode{ - ThreadList: threadListAssn, - Policy: model.WaitForThreadsPolicy_STOP_ON_FAILURE, + ThreadsToWaitFor: &model.WaitForThreadsNode_ThreadList{ + ThreadList: threadListAssn, + }, } nodeName, node := t.createBlankNode("threads", "WAIT_FOR_THREADS") diff --git a/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/CommonEnums.java b/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/CommonEnums.java index dfcfe1c5f..71fb5c8ee 100644 --- a/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/CommonEnums.java +++ b/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/CommonEnums.java @@ -41,10 +41,9 @@ public static void registerAllExtensions( "\020\001\022\026\n\022VAR_MUTATION_ERROR\020\002\022\027\n\023USER_TASK_" + "CANCELLED\020\003\022\013\n\007TIMEOUT\020\004\022\020\n\014TASK_FAILURE" + "\020\005\022\r\n\tVAR_ERROR\020\006\022\016\n\nTASK_ERROR\020\007\022\022\n\016INT" + - "ERNAL_ERROR\020\010*+\n\024WaitForThreadsPolicy\022\023\n" + - "\017STOP_ON_FAILURE\020\000BG\n\037io.littlehorse.sdk" + - ".common.protoP\001Z\007.;model\252\002\030LittleHorse.C" + - "ommon.Protob\006proto3" + "ERNAL_ERROR\020\010BG\n\037io.littlehorse.sdk.comm" + + "on.protoP\001Z\007.;model\252\002\030LittleHorse.Common" + + ".Protob\006proto3" }; descriptor = com.google.protobuf.Descriptors.FileDescriptor .internalBuildGeneratedFileFrom(descriptorData, diff --git a/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/NodeRunOuterClass.java b/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/NodeRunOuterClass.java index a77e127c8..6e7d4b551 100644 --- a/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/NodeRunOuterClass.java +++ b/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/NodeRunOuterClass.java @@ -116,29 +116,34 @@ public static void registerAllExtensions( "Run\"\\\n\016StartThreadRun\022\034\n\017child_thread_id" + "\030\001 \001(\005H\000\210\001\001\022\030\n\020thread_spec_name\030\002 \001(\tB\022\n" + "\020_child_thread_id\"3\n\027StartMultipleThread" + - "sRun\022\030\n\020thread_spec_name\030\001 \001(\t\"\307\002\n\021WaitF" + + "sRun\022\030\n\020thread_spec_name\030\001 \001(\t\"\244\004\n\021WaitF" + "orThreadsRun\022=\n\007threads\030\001 \003(\0132,.littleho" + - "rse.WaitForThreadsRun.WaitForThread\0221\n\006p" + - "olicy\030\002 \001(\0162!.littlehorse.WaitForThreads" + - "Policy\032\277\001\n\rWaitForThread\0228\n\017thread_end_t" + - "ime\030\001 \001(\0132\032.google.protobuf.TimestampH\000\210" + - "\001\001\022,\n\rthread_status\030\002 \001(\0162\025.littlehorse." + - "LHStatus\022\031\n\021thread_run_number\030\003 \001(\005\022\027\n\017a" + - "lready_handled\030\005 \001(\010B\022\n\020_thread_end_time" + - "\"\352\001\n\020ExternalEventRun\022>\n\025external_event_" + - "def_id\030\001 \001(\0132\037.littlehorse.ExternalEvent" + - "DefId\0223\n\nevent_time\030\002 \001(\0132\032.google.proto" + - "buf.TimestampH\000\210\001\001\022<\n\021external_event_id\030" + - "\003 \001(\0132\034.littlehorse.ExternalEventIdH\001\210\001\001" + - "B\r\n\013_event_timeB\024\n\022_external_event_id\"C\n" + - "\014SleepNodeRun\0223\n\017maturation_time\030\001 \001(\0132\032" + - ".google.protobuf.Timestamp\"\214\001\n\007Failure\022\024" + - "\n\014failure_name\030\001 \001(\t\022\017\n\007message\030\002 \001(\t\0220\n" + - "\007content\030\003 \001(\0132\032.littlehorse.VariableVal" + - "ueH\000\210\001\001\022\034\n\024was_properly_handled\030\004 \001(\010B\n\n" + - "\010_contentBG\n\037io.littlehorse.sdk.common.p" + - "rotoP\001Z\007.;model\252\002\030LittleHorse.Common.Pro" + - "tob\006proto3" + "rse.WaitForThreadsRun.WaitForThread\032\300\002\n\r" + + "WaitForThread\0228\n\017thread_end_time\030\001 \001(\0132\032" + + ".google.protobuf.TimestampH\000\210\001\001\022,\n\rthrea" + + "d_status\030\002 \001(\0162\025.littlehorse.LHStatus\022\031\n" + + "\021thread_run_number\030\003 \001(\005\022J\n\016waiting_stat" + + "us\030\004 \001(\01622.littlehorse.WaitForThreadsRun" + + ".WaitingThreadStatus\022*\n\035failure_handler_" + + "thread_run_id\030\005 \001(\005H\001\210\001\001B\022\n\020_thread_end_" + + "timeB \n\036_failure_handler_thread_run_id\"\214" + + "\001\n\023WaitingThreadStatus\022\026\n\022THREAD_IN_PROG" + + "RESS\020\000\022\033\n\027THREAD_HANDLING_FAILURE\020\001\022\'\n#T" + + "HREAD_COMPLETED_OR_FAILURE_HANDLED\020\002\022\027\n\023" + + "THREAD_UNSUCCESSFUL\020\003\"\352\001\n\020ExternalEventR" + + "un\022>\n\025external_event_def_id\030\001 \001(\0132\037.litt" + + "lehorse.ExternalEventDefId\0223\n\nevent_time" + + "\030\002 \001(\0132\032.google.protobuf.TimestampH\000\210\001\001\022" + + "<\n\021external_event_id\030\003 \001(\0132\034.littlehorse" + + ".ExternalEventIdH\001\210\001\001B\r\n\013_event_timeB\024\n\022" + + "_external_event_id\"C\n\014SleepNodeRun\0223\n\017ma" + + "turation_time\030\001 \001(\0132\032.google.protobuf.Ti" + + "mestamp\"\214\001\n\007Failure\022\024\n\014failure_name\030\001 \001(" + + "\t\022\017\n\007message\030\002 \001(\t\0220\n\007content\030\003 \001(\0132\032.li" + + "ttlehorse.VariableValueH\000\210\001\001\022\034\n\024was_prop" + + "erly_handled\030\004 \001(\010B\n\n\010_contentBG\n\037io.lit" + + "tlehorse.sdk.common.protoP\001Z\007.;model\252\002\030L" + + "ittleHorse.Common.Protob\006proto3" }; descriptor = com.google.protobuf.Descriptors.FileDescriptor .internalBuildGeneratedFileFrom(descriptorData, @@ -195,13 +200,13 @@ public static void registerAllExtensions( internal_static_littlehorse_WaitForThreadsRun_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( internal_static_littlehorse_WaitForThreadsRun_descriptor, - new java.lang.String[] { "Threads", "Policy", }); + new java.lang.String[] { "Threads", }); internal_static_littlehorse_WaitForThreadsRun_WaitForThread_descriptor = internal_static_littlehorse_WaitForThreadsRun_descriptor.getNestedTypes().get(0); internal_static_littlehorse_WaitForThreadsRun_WaitForThread_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( internal_static_littlehorse_WaitForThreadsRun_WaitForThread_descriptor, - new java.lang.String[] { "ThreadEndTime", "ThreadStatus", "ThreadRunNumber", "AlreadyHandled", "ThreadEndTime", }); + new java.lang.String[] { "ThreadEndTime", "ThreadStatus", "ThreadRunNumber", "WaitingStatus", "FailureHandlerThreadRunId", "ThreadEndTime", "FailureHandlerThreadRunId", }); internal_static_littlehorse_ExternalEventRun_descriptor = getDescriptor().getMessageTypes().get(8); internal_static_littlehorse_ExternalEventRun_fieldAccessorTable = new diff --git a/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsNode.java b/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsNode.java index 2c5041865..a4dae6706 100644 --- a/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsNode.java +++ b/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsNode.java @@ -16,8 +16,7 @@ private WaitForThreadsNode(com.google.protobuf.GeneratedMessageV3.Builder bui super(builder); } private WaitForThreadsNode() { - threads_ = java.util.Collections.emptyList(); - policy_ = 0; + perThreadFailureHandlers_ = java.util.Collections.emptyList(); } @java.lang.Override @@ -640,115 +639,931 @@ public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor getDef } - private int bitField0_; - public static final int THREADS_FIELD_NUMBER = 1; + public interface ThreadsToWaitForOrBuilder extends + // @@protoc_insertion_point(interface_extends:littlehorse.WaitForThreadsNode.ThreadsToWaitFor) + com.google.protobuf.MessageOrBuilder { + + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + java.util.List + getThreadsList(); + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor getThreads(int index); + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + int getThreadsCount(); + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + java.util.List + getThreadsOrBuilderList(); + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitForOrBuilder getThreadsOrBuilder( + int index); + } + /** + * Protobuf type {@code littlehorse.WaitForThreadsNode.ThreadsToWaitFor} + */ + public static final class ThreadsToWaitFor extends + com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:littlehorse.WaitForThreadsNode.ThreadsToWaitFor) + ThreadsToWaitForOrBuilder { + private static final long serialVersionUID = 0L; + // Use ThreadsToWaitFor.newBuilder() to construct. + private ThreadsToWaitFor(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + private ThreadsToWaitFor() { + threads_ = java.util.Collections.emptyList(); + } + + @java.lang.Override + @SuppressWarnings({"unused"}) + protected java.lang.Object newInstance( + UnusedPrivateParameter unused) { + return new ThreadsToWaitFor(); + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return io.littlehorse.sdk.common.proto.WfSpecOuterClass.internal_static_littlehorse_WaitForThreadsNode_ThreadsToWaitFor_descriptor; + } + + @java.lang.Override + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return io.littlehorse.sdk.common.proto.WfSpecOuterClass.internal_static_littlehorse_WaitForThreadsNode_ThreadsToWaitFor_fieldAccessorTable + .ensureFieldAccessorsInitialized( + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.class, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.Builder.class); + } + + public static final int THREADS_FIELD_NUMBER = 1; + @SuppressWarnings("serial") + private java.util.List threads_; + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + @java.lang.Override + public java.util.List getThreadsList() { + return threads_; + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + @java.lang.Override + public java.util.List + getThreadsOrBuilderList() { + return threads_; + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + @java.lang.Override + public int getThreadsCount() { + return threads_.size(); + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + @java.lang.Override + public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor getThreads(int index) { + return threads_.get(index); + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + @java.lang.Override + public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitForOrBuilder getThreadsOrBuilder( + int index) { + return threads_.get(index); + } + + private byte memoizedIsInitialized = -1; + @java.lang.Override + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) return true; + if (isInitialized == 0) return false; + + memoizedIsInitialized = 1; + return true; + } + + @java.lang.Override + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + for (int i = 0; i < threads_.size(); i++) { + output.writeMessage(1, threads_.get(i)); + } + getUnknownFields().writeTo(output); + } + + @java.lang.Override + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) return size; + + size = 0; + for (int i = 0; i < threads_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, threads_.get(i)); + } + size += getUnknownFields().getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor)) { + return super.equals(obj); + } + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor other = (io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor) obj; + + if (!getThreadsList() + .equals(other.getThreadsList())) return false; + if (!getUnknownFields().equals(other.getUnknownFields())) return false; + return true; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (getThreadsCount() > 0) { + hash = (37 * hash) + THREADS_FIELD_NUMBER; + hash = (53 * hash) + getThreadsList().hashCode(); + } + hash = (29 * hash) + getUnknownFields().hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor parseFrom( + java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor parseFrom( + java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + + public static io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input); + } + + public static io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input, extensionRegistry); + } + public static io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + + @java.lang.Override + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + public static Builder newBuilder(io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + @java.lang.Override + public Builder toBuilder() { + return this == DEFAULT_INSTANCE + ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code littlehorse.WaitForThreadsNode.ThreadsToWaitFor} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessageV3.Builder implements + // @@protoc_insertion_point(builder_implements:littlehorse.WaitForThreadsNode.ThreadsToWaitFor) + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitForOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return io.littlehorse.sdk.common.proto.WfSpecOuterClass.internal_static_littlehorse_WaitForThreadsNode_ThreadsToWaitFor_descriptor; + } + + @java.lang.Override + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return io.littlehorse.sdk.common.proto.WfSpecOuterClass.internal_static_littlehorse_WaitForThreadsNode_ThreadsToWaitFor_fieldAccessorTable + .ensureFieldAccessorsInitialized( + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.class, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.Builder.class); + } + + // Construct using io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.newBuilder() + private Builder() { + + } + + private Builder( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + + } + @java.lang.Override + public Builder clear() { + super.clear(); + bitField0_ = 0; + if (threadsBuilder_ == null) { + threads_ = java.util.Collections.emptyList(); + } else { + threads_ = null; + threadsBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + + @java.lang.Override + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return io.littlehorse.sdk.common.proto.WfSpecOuterClass.internal_static_littlehorse_WaitForThreadsNode_ThreadsToWaitFor_descriptor; + } + + @java.lang.Override + public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor getDefaultInstanceForType() { + return io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.getDefaultInstance(); + } + + @java.lang.Override + public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor build() { + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + @java.lang.Override + public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor buildPartial() { + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor result = new io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor(this); + buildPartialRepeatedFields(result); + if (bitField0_ != 0) { buildPartial0(result); } + onBuilt(); + return result; + } + + private void buildPartialRepeatedFields(io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor result) { + if (threadsBuilder_ == null) { + if (((bitField0_ & 0x00000001) != 0)) { + threads_ = java.util.Collections.unmodifiableList(threads_); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.threads_ = threads_; + } else { + result.threads_ = threadsBuilder_.build(); + } + } + + private void buildPartial0(io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor result) { + int from_bitField0_ = bitField0_; + } + + @java.lang.Override + public Builder clone() { + return super.clone(); + } + @java.lang.Override + public Builder setField( + com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return super.setField(field, value); + } + @java.lang.Override + public Builder clearField( + com.google.protobuf.Descriptors.FieldDescriptor field) { + return super.clearField(field); + } + @java.lang.Override + public Builder clearOneof( + com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return super.clearOneof(oneof); + } + @java.lang.Override + public Builder setRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + int index, java.lang.Object value) { + return super.setRepeatedField(field, index, value); + } + @java.lang.Override + public Builder addRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return super.addRepeatedField(field, value); + } + @java.lang.Override + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor) { + return mergeFrom((io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor other) { + if (other == io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.getDefaultInstance()) return this; + if (threadsBuilder_ == null) { + if (!other.threads_.isEmpty()) { + if (threads_.isEmpty()) { + threads_ = other.threads_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureThreadsIsMutable(); + threads_.addAll(other.threads_); + } + onChanged(); + } + } else { + if (!other.threads_.isEmpty()) { + if (threadsBuilder_.isEmpty()) { + threadsBuilder_.dispose(); + threadsBuilder_ = null; + threads_ = other.threads_; + bitField0_ = (bitField0_ & ~0x00000001); + threadsBuilder_ = + com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ? + getThreadsFieldBuilder() : null; + } else { + threadsBuilder_.addAllMessages(other.threads_); + } + } + } + this.mergeUnknownFields(other.getUnknownFields()); + onChanged(); + return this; + } + + @java.lang.Override + public final boolean isInitialized() { + return true; + } + + @java.lang.Override + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + case 10: { + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor m = + input.readMessage( + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.parser(), + extensionRegistry); + if (threadsBuilder_ == null) { + ensureThreadsIsMutable(); + threads_.add(m); + } else { + threadsBuilder_.addMessage(m); + } + break; + } // case 10 + default: { + if (!super.parseUnknownField(input, extensionRegistry, tag)) { + done = true; // was an endgroup tag + } + break; + } // default: + } // switch (tag) + } // while (!done) + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.unwrapIOException(); + } finally { + onChanged(); + } // finally + return this; + } + private int bitField0_; + + private java.util.List threads_ = + java.util.Collections.emptyList(); + private void ensureThreadsIsMutable() { + if (!((bitField0_ & 0x00000001) != 0)) { + threads_ = new java.util.ArrayList(threads_); + bitField0_ |= 0x00000001; + } + } + + private com.google.protobuf.RepeatedFieldBuilderV3< + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitForOrBuilder> threadsBuilder_; + + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public java.util.List getThreadsList() { + if (threadsBuilder_ == null) { + return java.util.Collections.unmodifiableList(threads_); + } else { + return threadsBuilder_.getMessageList(); + } + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public int getThreadsCount() { + if (threadsBuilder_ == null) { + return threads_.size(); + } else { + return threadsBuilder_.getCount(); + } + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor getThreads(int index) { + if (threadsBuilder_ == null) { + return threads_.get(index); + } else { + return threadsBuilder_.getMessage(index); + } + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public Builder setThreads( + int index, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor value) { + if (threadsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureThreadsIsMutable(); + threads_.set(index, value); + onChanged(); + } else { + threadsBuilder_.setMessage(index, value); + } + return this; + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public Builder setThreads( + int index, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder builderForValue) { + if (threadsBuilder_ == null) { + ensureThreadsIsMutable(); + threads_.set(index, builderForValue.build()); + onChanged(); + } else { + threadsBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public Builder addThreads(io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor value) { + if (threadsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureThreadsIsMutable(); + threads_.add(value); + onChanged(); + } else { + threadsBuilder_.addMessage(value); + } + return this; + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public Builder addThreads( + int index, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor value) { + if (threadsBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureThreadsIsMutable(); + threads_.add(index, value); + onChanged(); + } else { + threadsBuilder_.addMessage(index, value); + } + return this; + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public Builder addThreads( + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder builderForValue) { + if (threadsBuilder_ == null) { + ensureThreadsIsMutable(); + threads_.add(builderForValue.build()); + onChanged(); + } else { + threadsBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public Builder addThreads( + int index, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder builderForValue) { + if (threadsBuilder_ == null) { + ensureThreadsIsMutable(); + threads_.add(index, builderForValue.build()); + onChanged(); + } else { + threadsBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public Builder addAllThreads( + java.lang.Iterable values) { + if (threadsBuilder_ == null) { + ensureThreadsIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll( + values, threads_); + onChanged(); + } else { + threadsBuilder_.addAllMessages(values); + } + return this; + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public Builder clearThreads() { + if (threadsBuilder_ == null) { + threads_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + } else { + threadsBuilder_.clear(); + } + return this; + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public Builder removeThreads(int index) { + if (threadsBuilder_ == null) { + ensureThreadsIsMutable(); + threads_.remove(index); + onChanged(); + } else { + threadsBuilder_.remove(index); + } + return this; + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder getThreadsBuilder( + int index) { + return getThreadsFieldBuilder().getBuilder(index); + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitForOrBuilder getThreadsOrBuilder( + int index) { + if (threadsBuilder_ == null) { + return threads_.get(index); } else { + return threadsBuilder_.getMessageOrBuilder(index); + } + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public java.util.List + getThreadsOrBuilderList() { + if (threadsBuilder_ != null) { + return threadsBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(threads_); + } + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder addThreadsBuilder() { + return getThreadsFieldBuilder().addBuilder( + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.getDefaultInstance()); + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder addThreadsBuilder( + int index) { + return getThreadsFieldBuilder().addBuilder( + index, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.getDefaultInstance()); + } + /** + * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + */ + public java.util.List + getThreadsBuilderList() { + return getThreadsFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilderV3< + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitForOrBuilder> + getThreadsFieldBuilder() { + if (threadsBuilder_ == null) { + threadsBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3< + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitForOrBuilder>( + threads_, + ((bitField0_ & 0x00000001) != 0), + getParentForChildren(), + isClean()); + threads_ = null; + } + return threadsBuilder_; + } + @java.lang.Override + public final Builder setUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + @java.lang.Override + public final Builder mergeUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + + // @@protoc_insertion_point(builder_scope:littlehorse.WaitForThreadsNode.ThreadsToWaitFor) + } + + // @@protoc_insertion_point(class_scope:littlehorse.WaitForThreadsNode.ThreadsToWaitFor) + private static final io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor(); + } + + public static io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + private static final com.google.protobuf.Parser + PARSER = new com.google.protobuf.AbstractParser() { + @java.lang.Override + public ThreadsToWaitFor parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + Builder builder = newBuilder(); + try { + builder.mergeFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(builder.buildPartial()); + } catch (com.google.protobuf.UninitializedMessageException e) { + throw e.asInvalidProtocolBufferException().setUnfinishedMessage(builder.buildPartial()); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e) + .setUnfinishedMessage(builder.buildPartial()); + } + return builder.buildPartial(); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + @java.lang.Override + public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + private int threadsToWaitForCase_ = 0; @SuppressWarnings("serial") - private java.util.List threads_; + private java.lang.Object threadsToWaitFor_; + public enum ThreadsToWaitForCase + implements com.google.protobuf.Internal.EnumLite, + com.google.protobuf.AbstractMessage.InternalOneOfEnum { + THREADS(1), + THREAD_LIST(2), + THREADSTOWAITFOR_NOT_SET(0); + private final int value; + private ThreadsToWaitForCase(int value) { + this.value = value; + } + /** + * @param value The number of the enum to look for. + * @return The enum associated with the given number. + * @deprecated Use {@link #forNumber(int)} instead. + */ + @java.lang.Deprecated + public static ThreadsToWaitForCase valueOf(int value) { + return forNumber(value); + } + + public static ThreadsToWaitForCase forNumber(int value) { + switch (value) { + case 1: return THREADS; + case 2: return THREAD_LIST; + case 0: return THREADSTOWAITFOR_NOT_SET; + default: return null; + } + } + public int getNumber() { + return this.value; + } + }; + + public ThreadsToWaitForCase + getThreadsToWaitForCase() { + return ThreadsToWaitForCase.forNumber( + threadsToWaitForCase_); + } + + public static final int THREADS_FIELD_NUMBER = 1; /** - * 
-   * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-   * for compatibility reasons, we cannot wrap it into a separate message.
-   *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.WaitForThreadsNode.ThreadsToWaitFor threads = 1; + * @return Whether the threads field is set. */ @java.lang.Override - public java.util.List getThreadsList() { - return threads_; + public boolean hasThreads() { + return threadsToWaitForCase_ == 1; } /** - * 
-   * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-   * for compatibility reasons, we cannot wrap it into a separate message.
-   *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.WaitForThreadsNode.ThreadsToWaitFor threads = 1; + * @return The threads. */ @java.lang.Override - public java.util.List - getThreadsOrBuilderList() { - return threads_; + public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor getThreads() { + if (threadsToWaitForCase_ == 1) { + return (io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor) threadsToWaitFor_; + } + return io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.getDefaultInstance(); } /** - * 
-   * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-   * for compatibility reasons, we cannot wrap it into a separate message.
-   *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.WaitForThreadsNode.ThreadsToWaitFor threads = 1; */ @java.lang.Override - public int getThreadsCount() { - return threads_.size(); + public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitForOrBuilder getThreadsOrBuilder() { + if (threadsToWaitForCase_ == 1) { + return (io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor) threadsToWaitFor_; + } + return io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.getDefaultInstance(); } + + public static final int THREAD_LIST_FIELD_NUMBER = 2; /** - * 
-   * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-   * for compatibility reasons, we cannot wrap it into a separate message.
-   *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.VariableAssignment thread_list = 2; + * @return Whether the threadList field is set. */ @java.lang.Override - public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor getThreads(int index) { - return threads_.get(index); + public boolean hasThreadList() { + return threadsToWaitForCase_ == 2; } /** - * 
-   * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-   * for compatibility reasons, we cannot wrap it into a separate message.
-   *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.VariableAssignment thread_list = 2; + * @return The threadList. */ @java.lang.Override - public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitForOrBuilder getThreadsOrBuilder( - int index) { - return threads_.get(index); + public io.littlehorse.sdk.common.proto.VariableAssignment getThreadList() { + if (threadsToWaitForCase_ == 2) { + return (io.littlehorse.sdk.common.proto.VariableAssignment) threadsToWaitFor_; + } + return io.littlehorse.sdk.common.proto.VariableAssignment.getDefaultInstance(); + } + /** + * .littlehorse.VariableAssignment thread_list = 2; + */ + @java.lang.Override + public io.littlehorse.sdk.common.proto.VariableAssignmentOrBuilder getThreadListOrBuilder() { + if (threadsToWaitForCase_ == 2) { + return (io.littlehorse.sdk.common.proto.VariableAssignment) threadsToWaitFor_; + } + return io.littlehorse.sdk.common.proto.VariableAssignment.getDefaultInstance(); } - public static final int THREAD_LIST_FIELD_NUMBER = 3; - private io.littlehorse.sdk.common.proto.VariableAssignment threadList_; + public static final int PER_THREAD_FAILURE_HANDLERS_FIELD_NUMBER = 3; + @SuppressWarnings("serial") + private java.util.List perThreadFailureHandlers_; /** - * optional .littlehorse.VariableAssignment thread_list = 3; - * @return Whether the threadList field is set. + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; */ @java.lang.Override - public boolean hasThreadList() { - return ((bitField0_ & 0x00000001) != 0); + public java.util.List getPerThreadFailureHandlersList() { + return perThreadFailureHandlers_; } /** - * optional .littlehorse.VariableAssignment thread_list = 3; - * @return The threadList. + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; */ @java.lang.Override - public io.littlehorse.sdk.common.proto.VariableAssignment getThreadList() { - return threadList_ == null ? io.littlehorse.sdk.common.proto.VariableAssignment.getDefaultInstance() : threadList_; + public java.util.List + getPerThreadFailureHandlersOrBuilderList() { + return perThreadFailureHandlers_; } /** - * optional .littlehorse.VariableAssignment thread_list = 3; + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; */ @java.lang.Override - public io.littlehorse.sdk.common.proto.VariableAssignmentOrBuilder getThreadListOrBuilder() { - return threadList_ == null ? io.littlehorse.sdk.common.proto.VariableAssignment.getDefaultInstance() : threadList_; + public int getPerThreadFailureHandlersCount() { + return perThreadFailureHandlers_.size(); } - - public static final int POLICY_FIELD_NUMBER = 2; - private int policy_ = 0; /** - * .littlehorse.WaitForThreadsPolicy policy = 2; - * @return The enum numeric value on the wire for policy. + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; */ - @java.lang.Override public int getPolicyValue() { - return policy_; + @java.lang.Override + public io.littlehorse.sdk.common.proto.FailureHandlerDef getPerThreadFailureHandlers(int index) { + return perThreadFailureHandlers_.get(index); } /** - * .littlehorse.WaitForThreadsPolicy policy = 2; - * @return The policy. + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; */ - @java.lang.Override public io.littlehorse.sdk.common.proto.WaitForThreadsPolicy getPolicy() { - io.littlehorse.sdk.common.proto.WaitForThreadsPolicy result = io.littlehorse.sdk.common.proto.WaitForThreadsPolicy.forNumber(policy_); - return result == null ? io.littlehorse.sdk.common.proto.WaitForThreadsPolicy.UNRECOGNIZED : result; + @java.lang.Override + public io.littlehorse.sdk.common.proto.FailureHandlerDefOrBuilder getPerThreadFailureHandlersOrBuilder( + int index) { + return perThreadFailureHandlers_.get(index); } private byte memoizedIsInitialized = -1; @@ -765,14 +1580,14 @@ public final boolean isInitialized() { @java.lang.Override public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { - for (int i = 0; i < threads_.size(); i++) { - output.writeMessage(1, threads_.get(i)); + if (threadsToWaitForCase_ == 1) { + output.writeMessage(1, (io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor) threadsToWaitFor_); } - if (policy_ != io.littlehorse.sdk.common.proto.WaitForThreadsPolicy.STOP_ON_FAILURE.getNumber()) { - output.writeEnum(2, policy_); + if (threadsToWaitForCase_ == 2) { + output.writeMessage(2, (io.littlehorse.sdk.common.proto.VariableAssignment) threadsToWaitFor_); } - if (((bitField0_ & 0x00000001) != 0)) { - output.writeMessage(3, getThreadList()); + for (int i = 0; i < perThreadFailureHandlers_.size(); i++) { + output.writeMessage(3, perThreadFailureHandlers_.get(i)); } getUnknownFields().writeTo(output); } @@ -783,17 +1598,17 @@ public int getSerializedSize() { if (size != -1) return size; size = 0; - for (int i = 0; i < threads_.size(); i++) { + if (threadsToWaitForCase_ == 1) { size += com.google.protobuf.CodedOutputStream - .computeMessageSize(1, threads_.get(i)); + .computeMessageSize(1, (io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor) threadsToWaitFor_); } - if (policy_ != io.littlehorse.sdk.common.proto.WaitForThreadsPolicy.STOP_ON_FAILURE.getNumber()) { + if (threadsToWaitForCase_ == 2) { size += com.google.protobuf.CodedOutputStream - .computeEnumSize(2, policy_); + .computeMessageSize(2, (io.littlehorse.sdk.common.proto.VariableAssignment) threadsToWaitFor_); } - if (((bitField0_ & 0x00000001) != 0)) { + for (int i = 0; i < perThreadFailureHandlers_.size(); i++) { size += com.google.protobuf.CodedOutputStream - .computeMessageSize(3, getThreadList()); + .computeMessageSize(3, perThreadFailureHandlers_.get(i)); } size += getUnknownFields().getSerializedSize(); memoizedSize = size; @@ -810,14 +1625,21 @@ public boolean equals(final java.lang.Object obj) { } io.littlehorse.sdk.common.proto.WaitForThreadsNode other = (io.littlehorse.sdk.common.proto.WaitForThreadsNode) obj; - if (!getThreadsList() - .equals(other.getThreadsList())) return false; - if (hasThreadList() != other.hasThreadList()) return false; - if (hasThreadList()) { - if (!getThreadList() - .equals(other.getThreadList())) return false; + if (!getPerThreadFailureHandlersList() + .equals(other.getPerThreadFailureHandlersList())) return false; + if (!getThreadsToWaitForCase().equals(other.getThreadsToWaitForCase())) return false; + switch (threadsToWaitForCase_) { + case 1: + if (!getThreads() + .equals(other.getThreads())) return false; + break; + case 2: + if (!getThreadList() + .equals(other.getThreadList())) return false; + break; + case 0: + default: } - if (policy_ != other.policy_) return false; if (!getUnknownFields().equals(other.getUnknownFields())) return false; return true; } @@ -829,16 +1651,22 @@ public int hashCode() { } int hash = 41; hash = (19 * hash) + getDescriptor().hashCode(); - if (getThreadsCount() > 0) { - hash = (37 * hash) + THREADS_FIELD_NUMBER; - hash = (53 * hash) + getThreadsList().hashCode(); - } - if (hasThreadList()) { - hash = (37 * hash) + THREAD_LIST_FIELD_NUMBER; - hash = (53 * hash) + getThreadList().hashCode(); + if (getPerThreadFailureHandlersCount() > 0) { + hash = (37 * hash) + PER_THREAD_FAILURE_HANDLERS_FIELD_NUMBER; + hash = (53 * hash) + getPerThreadFailureHandlersList().hashCode(); + } + switch (threadsToWaitForCase_) { + case 1: + hash = (37 * hash) + THREADS_FIELD_NUMBER; + hash = (53 * hash) + getThreads().hashCode(); + break; + case 2: + hash = (37 * hash) + THREAD_LIST_FIELD_NUMBER; + hash = (53 * hash) + getThreadList().hashCode(); + break; + case 0: + default: } - hash = (37 * hash) + POLICY_FIELD_NUMBER; - hash = (53 * hash) + policy_; hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -958,38 +1786,33 @@ public static final class Builder extends // Construct using io.littlehorse.sdk.common.proto.WaitForThreadsNode.newBuilder() private Builder() { - maybeForceBuilderInitialization(); + } private Builder( com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { super(parent); - maybeForceBuilderInitialization(); - } - private void maybeForceBuilderInitialization() { - if (com.google.protobuf.GeneratedMessageV3 - .alwaysUseFieldBuilders) { - getThreadsFieldBuilder(); - getThreadListFieldBuilder(); - } + } @java.lang.Override public Builder clear() { super.clear(); bitField0_ = 0; - if (threadsBuilder_ == null) { - threads_ = java.util.Collections.emptyList(); - } else { - threads_ = null; + if (threadsBuilder_ != null) { threadsBuilder_.clear(); } - bitField0_ = (bitField0_ & ~0x00000001); - threadList_ = null; if (threadListBuilder_ != null) { - threadListBuilder_.dispose(); - threadListBuilder_ = null; + threadListBuilder_.clear(); + } + if (perThreadFailureHandlersBuilder_ == null) { + perThreadFailureHandlers_ = java.util.Collections.emptyList(); + } else { + perThreadFailureHandlers_ = null; + perThreadFailureHandlersBuilder_.clear(); } - policy_ = 0; + bitField0_ = (bitField0_ & ~0x00000004); + threadsToWaitForCase_ = 0; + threadsToWaitFor_ = null; return this; } @@ -1018,35 +1841,38 @@ public io.littlehorse.sdk.common.proto.WaitForThreadsNode buildPartial() { io.littlehorse.sdk.common.proto.WaitForThreadsNode result = new io.littlehorse.sdk.common.proto.WaitForThreadsNode(this); buildPartialRepeatedFields(result); if (bitField0_ != 0) { buildPartial0(result); } + buildPartialOneofs(result); onBuilt(); return result; } private void buildPartialRepeatedFields(io.littlehorse.sdk.common.proto.WaitForThreadsNode result) { - if (threadsBuilder_ == null) { - if (((bitField0_ & 0x00000001) != 0)) { - threads_ = java.util.Collections.unmodifiableList(threads_); - bitField0_ = (bitField0_ & ~0x00000001); + if (perThreadFailureHandlersBuilder_ == null) { + if (((bitField0_ & 0x00000004) != 0)) { + perThreadFailureHandlers_ = java.util.Collections.unmodifiableList(perThreadFailureHandlers_); + bitField0_ = (bitField0_ & ~0x00000004); } - result.threads_ = threads_; + result.perThreadFailureHandlers_ = perThreadFailureHandlers_; } else { - result.threads_ = threadsBuilder_.build(); + result.perThreadFailureHandlers_ = perThreadFailureHandlersBuilder_.build(); } } private void buildPartial0(io.littlehorse.sdk.common.proto.WaitForThreadsNode result) { int from_bitField0_ = bitField0_; - int to_bitField0_ = 0; - if (((from_bitField0_ & 0x00000002) != 0)) { - result.threadList_ = threadListBuilder_ == null - ? threadList_ - : threadListBuilder_.build(); - to_bitField0_ |= 0x00000001; + } + + private void buildPartialOneofs(io.littlehorse.sdk.common.proto.WaitForThreadsNode result) { + result.threadsToWaitForCase_ = threadsToWaitForCase_; + result.threadsToWaitFor_ = this.threadsToWaitFor_; + if (threadsToWaitForCase_ == 1 && + threadsBuilder_ != null) { + result.threadsToWaitFor_ = threadsBuilder_.build(); } - if (((from_bitField0_ & 0x00000004) != 0)) { - result.policy_ = policy_; + if (threadsToWaitForCase_ == 2 && + threadListBuilder_ != null) { + result.threadsToWaitFor_ = threadListBuilder_.build(); } - result.bitField0_ |= to_bitField0_; } @java.lang.Override @@ -1093,37 +1919,44 @@ public Builder mergeFrom(com.google.protobuf.Message other) { public Builder mergeFrom(io.littlehorse.sdk.common.proto.WaitForThreadsNode other) { if (other == io.littlehorse.sdk.common.proto.WaitForThreadsNode.getDefaultInstance()) return this; - if (threadsBuilder_ == null) { - if (!other.threads_.isEmpty()) { - if (threads_.isEmpty()) { - threads_ = other.threads_; - bitField0_ = (bitField0_ & ~0x00000001); + if (perThreadFailureHandlersBuilder_ == null) { + if (!other.perThreadFailureHandlers_.isEmpty()) { + if (perThreadFailureHandlers_.isEmpty()) { + perThreadFailureHandlers_ = other.perThreadFailureHandlers_; + bitField0_ = (bitField0_ & ~0x00000004); } else { - ensureThreadsIsMutable(); - threads_.addAll(other.threads_); + ensurePerThreadFailureHandlersIsMutable(); + perThreadFailureHandlers_.addAll(other.perThreadFailureHandlers_); } onChanged(); } } else { - if (!other.threads_.isEmpty()) { - if (threadsBuilder_.isEmpty()) { - threadsBuilder_.dispose(); - threadsBuilder_ = null; - threads_ = other.threads_; - bitField0_ = (bitField0_ & ~0x00000001); - threadsBuilder_ = + if (!other.perThreadFailureHandlers_.isEmpty()) { + if (perThreadFailureHandlersBuilder_.isEmpty()) { + perThreadFailureHandlersBuilder_.dispose(); + perThreadFailureHandlersBuilder_ = null; + perThreadFailureHandlers_ = other.perThreadFailureHandlers_; + bitField0_ = (bitField0_ & ~0x00000004); + perThreadFailureHandlersBuilder_ = com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ? - getThreadsFieldBuilder() : null; + getPerThreadFailureHandlersFieldBuilder() : null; } else { - threadsBuilder_.addAllMessages(other.threads_); + perThreadFailureHandlersBuilder_.addAllMessages(other.perThreadFailureHandlers_); } } } - if (other.hasThreadList()) { - mergeThreadList(other.getThreadList()); - } - if (other.policy_ != 0) { - setPolicyValue(other.getPolicyValue()); + switch (other.getThreadsToWaitForCase()) { + case THREADS: { + mergeThreads(other.getThreads()); + break; + } + case THREAD_LIST: { + mergeThreadList(other.getThreadList()); + break; + } + case THREADSTOWAITFOR_NOT_SET: { + break; + } } this.mergeUnknownFields(other.getUnknownFields()); onChanged(); @@ -1152,28 +1985,30 @@ public Builder mergeFrom( done = true; break; case 10: { - io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor m = - input.readMessage( - io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.parser(), - extensionRegistry); - if (threadsBuilder_ == null) { - ensureThreadsIsMutable(); - threads_.add(m); - } else { - threadsBuilder_.addMessage(m); - } + input.readMessage( + getThreadsFieldBuilder().getBuilder(), + extensionRegistry); + threadsToWaitForCase_ = 1; break; } // case 10 - case 16: { - policy_ = input.readEnum(); - bitField0_ |= 0x00000004; - break; - } // case 16 - case 26: { + case 18: { input.readMessage( getThreadListFieldBuilder().getBuilder(), extensionRegistry); - bitField0_ |= 0x00000002; + threadsToWaitForCase_ = 2; + break; + } // case 18 + case 26: { + io.littlehorse.sdk.common.proto.FailureHandlerDef m = + input.readMessage( + io.littlehorse.sdk.common.proto.FailureHandlerDef.parser(), + extensionRegistry); + if (perThreadFailureHandlersBuilder_ == null) { + ensurePerThreadFailureHandlersIsMutable(); + perThreadFailureHandlers_.add(m); + } else { + perThreadFailureHandlersBuilder_.addMessage(m); + } break; } // case 26 default: { @@ -1191,509 +2026,546 @@ public Builder mergeFrom( } // finally return this; } - private int bitField0_; + private int threadsToWaitForCase_ = 0; + private java.lang.Object threadsToWaitFor_; + public ThreadsToWaitForCase + getThreadsToWaitForCase() { + return ThreadsToWaitForCase.forNumber( + threadsToWaitForCase_); + } - private java.util.List threads_ = - java.util.Collections.emptyList(); - private void ensureThreadsIsMutable() { - if (!((bitField0_ & 0x00000001) != 0)) { - threads_ = new java.util.ArrayList(threads_); - bitField0_ |= 0x00000001; - } + public Builder clearThreadsToWaitFor() { + threadsToWaitForCase_ = 0; + threadsToWaitFor_ = null; + onChanged(); + return this; } - private com.google.protobuf.RepeatedFieldBuilderV3< - io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitForOrBuilder> threadsBuilder_; + private int bitField0_; + private com.google.protobuf.SingleFieldBuilderV3< + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.Builder, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitForOrBuilder> threadsBuilder_; /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; - */ - public java.util.List getThreadsList() { - if (threadsBuilder_ == null) { - return java.util.Collections.unmodifiableList(threads_); - } else { - return threadsBuilder_.getMessageList(); - } - } - /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.WaitForThreadsNode.ThreadsToWaitFor threads = 1; + * @return Whether the threads field is set. */ - public int getThreadsCount() { - if (threadsBuilder_ == null) { - return threads_.size(); - } else { - return threadsBuilder_.getCount(); - } + @java.lang.Override + public boolean hasThreads() { + return threadsToWaitForCase_ == 1; } /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.WaitForThreadsNode.ThreadsToWaitFor threads = 1; + * @return The threads. */ - public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor getThreads(int index) { + @java.lang.Override + public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor getThreads() { if (threadsBuilder_ == null) { - return threads_.get(index); + if (threadsToWaitForCase_ == 1) { + return (io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor) threadsToWaitFor_; + } + return io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.getDefaultInstance(); } else { - return threadsBuilder_.getMessage(index); + if (threadsToWaitForCase_ == 1) { + return threadsBuilder_.getMessage(); + } + return io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.getDefaultInstance(); } } /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.WaitForThreadsNode.ThreadsToWaitFor threads = 1; */ - public Builder setThreads( - int index, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor value) { + public Builder setThreads(io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor value) { if (threadsBuilder_ == null) { if (value == null) { throw new NullPointerException(); } - ensureThreadsIsMutable(); - threads_.set(index, value); + threadsToWaitFor_ = value; onChanged(); } else { - threadsBuilder_.setMessage(index, value); + threadsBuilder_.setMessage(value); } + threadsToWaitForCase_ = 1; return this; } /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.WaitForThreadsNode.ThreadsToWaitFor threads = 1; */ public Builder setThreads( - int index, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder builderForValue) { + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.Builder builderForValue) { if (threadsBuilder_ == null) { - ensureThreadsIsMutable(); - threads_.set(index, builderForValue.build()); + threadsToWaitFor_ = builderForValue.build(); onChanged(); } else { - threadsBuilder_.setMessage(index, builderForValue.build()); + threadsBuilder_.setMessage(builderForValue.build()); } + threadsToWaitForCase_ = 1; return this; } /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.WaitForThreadsNode.ThreadsToWaitFor threads = 1; */ - public Builder addThreads(io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor value) { + public Builder mergeThreads(io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor value) { if (threadsBuilder_ == null) { - if (value == null) { - throw new NullPointerException(); + if (threadsToWaitForCase_ == 1 && + threadsToWaitFor_ != io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.getDefaultInstance()) { + threadsToWaitFor_ = io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.newBuilder((io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor) threadsToWaitFor_) + .mergeFrom(value).buildPartial(); + } else { + threadsToWaitFor_ = value; } - ensureThreadsIsMutable(); - threads_.add(value); onChanged(); } else { - threadsBuilder_.addMessage(value); - } - return this; - } - /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; - */ - public Builder addThreads( - int index, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor value) { - if (threadsBuilder_ == null) { - if (value == null) { - throw new NullPointerException(); + if (threadsToWaitForCase_ == 1) { + threadsBuilder_.mergeFrom(value); + } else { + threadsBuilder_.setMessage(value); } - ensureThreadsIsMutable(); - threads_.add(index, value); - onChanged(); - } else { - threadsBuilder_.addMessage(index, value); - } - return this; - } - /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; - */ - public Builder addThreads( - io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder builderForValue) { - if (threadsBuilder_ == null) { - ensureThreadsIsMutable(); - threads_.add(builderForValue.build()); - onChanged(); - } else { - threadsBuilder_.addMessage(builderForValue.build()); - } - return this; - } - /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; - */ - public Builder addThreads( - int index, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder builderForValue) { - if (threadsBuilder_ == null) { - ensureThreadsIsMutable(); - threads_.add(index, builderForValue.build()); - onChanged(); - } else { - threadsBuilder_.addMessage(index, builderForValue.build()); - } - return this; - } - /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; - */ - public Builder addAllThreads( - java.lang.Iterable values) { - if (threadsBuilder_ == null) { - ensureThreadsIsMutable(); - com.google.protobuf.AbstractMessageLite.Builder.addAll( - values, threads_); - onChanged(); - } else { - threadsBuilder_.addAllMessages(values); } + threadsToWaitForCase_ = 1; return this; } /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.WaitForThreadsNode.ThreadsToWaitFor threads = 1; */ public Builder clearThreads() { if (threadsBuilder_ == null) { - threads_ = java.util.Collections.emptyList(); - bitField0_ = (bitField0_ & ~0x00000001); - onChanged(); + if (threadsToWaitForCase_ == 1) { + threadsToWaitForCase_ = 0; + threadsToWaitFor_ = null; + onChanged(); + } } else { + if (threadsToWaitForCase_ == 1) { + threadsToWaitForCase_ = 0; + threadsToWaitFor_ = null; + } threadsBuilder_.clear(); } return this; } /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; - */ - public Builder removeThreads(int index) { - if (threadsBuilder_ == null) { - ensureThreadsIsMutable(); - threads_.remove(index); - onChanged(); - } else { - threadsBuilder_.remove(index); - } - return this; - } - /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; - */ - public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder getThreadsBuilder( - int index) { - return getThreadsFieldBuilder().getBuilder(index); - } - /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.WaitForThreadsNode.ThreadsToWaitFor threads = 1; */ - public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitForOrBuilder getThreadsOrBuilder( - int index) { - if (threadsBuilder_ == null) { - return threads_.get(index); } else { - return threadsBuilder_.getMessageOrBuilder(index); - } + public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.Builder getThreadsBuilder() { + return getThreadsFieldBuilder().getBuilder(); } /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.WaitForThreadsNode.ThreadsToWaitFor threads = 1; */ - public java.util.List - getThreadsOrBuilderList() { - if (threadsBuilder_ != null) { - return threadsBuilder_.getMessageOrBuilderList(); + @java.lang.Override + public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitForOrBuilder getThreadsOrBuilder() { + if ((threadsToWaitForCase_ == 1) && (threadsBuilder_ != null)) { + return threadsBuilder_.getMessageOrBuilder(); } else { - return java.util.Collections.unmodifiableList(threads_); + if (threadsToWaitForCase_ == 1) { + return (io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor) threadsToWaitFor_; + } + return io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.getDefaultInstance(); } } /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; - */ - public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder addThreadsBuilder() { - return getThreadsFieldBuilder().addBuilder( - io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.getDefaultInstance()); - } - /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; - */ - public io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder addThreadsBuilder( - int index) { - return getThreadsFieldBuilder().addBuilder( - index, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.getDefaultInstance()); - } - /** - * 
-     * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-     * for compatibility reasons, we cannot wrap it into a separate message.
-     *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.WaitForThreadsNode.ThreadsToWaitFor threads = 1; */ - public java.util.List - getThreadsBuilderList() { - return getThreadsFieldBuilder().getBuilderList(); - } - private com.google.protobuf.RepeatedFieldBuilderV3< - io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitForOrBuilder> + private com.google.protobuf.SingleFieldBuilderV3< + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.Builder, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitForOrBuilder> getThreadsFieldBuilder() { if (threadsBuilder_ == null) { - threadsBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3< - io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor.Builder, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitForOrBuilder>( - threads_, - ((bitField0_ & 0x00000001) != 0), + if (!(threadsToWaitForCase_ == 1)) { + threadsToWaitFor_ = io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.getDefaultInstance(); + } + threadsBuilder_ = new com.google.protobuf.SingleFieldBuilderV3< + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor.Builder, io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitForOrBuilder>( + (io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor) threadsToWaitFor_, getParentForChildren(), isClean()); - threads_ = null; + threadsToWaitFor_ = null; } + threadsToWaitForCase_ = 1; + onChanged(); return threadsBuilder_; } - private io.littlehorse.sdk.common.proto.VariableAssignment threadList_; private com.google.protobuf.SingleFieldBuilderV3< io.littlehorse.sdk.common.proto.VariableAssignment, io.littlehorse.sdk.common.proto.VariableAssignment.Builder, io.littlehorse.sdk.common.proto.VariableAssignmentOrBuilder> threadListBuilder_; /** - * optional .littlehorse.VariableAssignment thread_list = 3; + * .littlehorse.VariableAssignment thread_list = 2; * @return Whether the threadList field is set. */ + @java.lang.Override public boolean hasThreadList() { - return ((bitField0_ & 0x00000002) != 0); + return threadsToWaitForCase_ == 2; } /** - * optional .littlehorse.VariableAssignment thread_list = 3; + * .littlehorse.VariableAssignment thread_list = 2; * @return The threadList. */ + @java.lang.Override public io.littlehorse.sdk.common.proto.VariableAssignment getThreadList() { if (threadListBuilder_ == null) { - return threadList_ == null ? io.littlehorse.sdk.common.proto.VariableAssignment.getDefaultInstance() : threadList_; + if (threadsToWaitForCase_ == 2) { + return (io.littlehorse.sdk.common.proto.VariableAssignment) threadsToWaitFor_; + } + return io.littlehorse.sdk.common.proto.VariableAssignment.getDefaultInstance(); } else { - return threadListBuilder_.getMessage(); + if (threadsToWaitForCase_ == 2) { + return threadListBuilder_.getMessage(); + } + return io.littlehorse.sdk.common.proto.VariableAssignment.getDefaultInstance(); } } /** - * optional .littlehorse.VariableAssignment thread_list = 3; + * .littlehorse.VariableAssignment thread_list = 2; */ public Builder setThreadList(io.littlehorse.sdk.common.proto.VariableAssignment value) { if (threadListBuilder_ == null) { if (value == null) { throw new NullPointerException(); } - threadList_ = value; + threadsToWaitFor_ = value; + onChanged(); } else { threadListBuilder_.setMessage(value); } - bitField0_ |= 0x00000002; - onChanged(); + threadsToWaitForCase_ = 2; return this; } /** - * optional .littlehorse.VariableAssignment thread_list = 3; + * .littlehorse.VariableAssignment thread_list = 2; */ public Builder setThreadList( io.littlehorse.sdk.common.proto.VariableAssignment.Builder builderForValue) { if (threadListBuilder_ == null) { - threadList_ = builderForValue.build(); + threadsToWaitFor_ = builderForValue.build(); + onChanged(); } else { threadListBuilder_.setMessage(builderForValue.build()); } - bitField0_ |= 0x00000002; - onChanged(); + threadsToWaitForCase_ = 2; return this; } /** - * optional .littlehorse.VariableAssignment thread_list = 3; + * .littlehorse.VariableAssignment thread_list = 2; */ public Builder mergeThreadList(io.littlehorse.sdk.common.proto.VariableAssignment value) { if (threadListBuilder_ == null) { - if (((bitField0_ & 0x00000002) != 0) && - threadList_ != null && - threadList_ != io.littlehorse.sdk.common.proto.VariableAssignment.getDefaultInstance()) { - getThreadListBuilder().mergeFrom(value); + if (threadsToWaitForCase_ == 2 && + threadsToWaitFor_ != io.littlehorse.sdk.common.proto.VariableAssignment.getDefaultInstance()) { + threadsToWaitFor_ = io.littlehorse.sdk.common.proto.VariableAssignment.newBuilder((io.littlehorse.sdk.common.proto.VariableAssignment) threadsToWaitFor_) + .mergeFrom(value).buildPartial(); } else { - threadList_ = value; + threadsToWaitFor_ = value; } + onChanged(); } else { - threadListBuilder_.mergeFrom(value); + if (threadsToWaitForCase_ == 2) { + threadListBuilder_.mergeFrom(value); + } else { + threadListBuilder_.setMessage(value); + } } - bitField0_ |= 0x00000002; - onChanged(); + threadsToWaitForCase_ = 2; return this; } /** - * optional .littlehorse.VariableAssignment thread_list = 3; + * .littlehorse.VariableAssignment thread_list = 2; */ public Builder clearThreadList() { - bitField0_ = (bitField0_ & ~0x00000002); - threadList_ = null; - if (threadListBuilder_ != null) { - threadListBuilder_.dispose(); - threadListBuilder_ = null; + if (threadListBuilder_ == null) { + if (threadsToWaitForCase_ == 2) { + threadsToWaitForCase_ = 0; + threadsToWaitFor_ = null; + onChanged(); + } + } else { + if (threadsToWaitForCase_ == 2) { + threadsToWaitForCase_ = 0; + threadsToWaitFor_ = null; + } + threadListBuilder_.clear(); } - onChanged(); return this; } /** - * optional .littlehorse.VariableAssignment thread_list = 3; + * .littlehorse.VariableAssignment thread_list = 2; */ public io.littlehorse.sdk.common.proto.VariableAssignment.Builder getThreadListBuilder() { - bitField0_ |= 0x00000002; - onChanged(); return getThreadListFieldBuilder().getBuilder(); } /** - * optional .littlehorse.VariableAssignment thread_list = 3; + * .littlehorse.VariableAssignment thread_list = 2; */ + @java.lang.Override public io.littlehorse.sdk.common.proto.VariableAssignmentOrBuilder getThreadListOrBuilder() { - if (threadListBuilder_ != null) { + if ((threadsToWaitForCase_ == 2) && (threadListBuilder_ != null)) { return threadListBuilder_.getMessageOrBuilder(); } else { - return threadList_ == null ? - io.littlehorse.sdk.common.proto.VariableAssignment.getDefaultInstance() : threadList_; + if (threadsToWaitForCase_ == 2) { + return (io.littlehorse.sdk.common.proto.VariableAssignment) threadsToWaitFor_; + } + return io.littlehorse.sdk.common.proto.VariableAssignment.getDefaultInstance(); } } /** - * optional .littlehorse.VariableAssignment thread_list = 3; + * .littlehorse.VariableAssignment thread_list = 2; */ private com.google.protobuf.SingleFieldBuilderV3< io.littlehorse.sdk.common.proto.VariableAssignment, io.littlehorse.sdk.common.proto.VariableAssignment.Builder, io.littlehorse.sdk.common.proto.VariableAssignmentOrBuilder> getThreadListFieldBuilder() { if (threadListBuilder_ == null) { + if (!(threadsToWaitForCase_ == 2)) { + threadsToWaitFor_ = io.littlehorse.sdk.common.proto.VariableAssignment.getDefaultInstance(); + } threadListBuilder_ = new com.google.protobuf.SingleFieldBuilderV3< io.littlehorse.sdk.common.proto.VariableAssignment, io.littlehorse.sdk.common.proto.VariableAssignment.Builder, io.littlehorse.sdk.common.proto.VariableAssignmentOrBuilder>( - getThreadList(), + (io.littlehorse.sdk.common.proto.VariableAssignment) threadsToWaitFor_, getParentForChildren(), isClean()); - threadList_ = null; + threadsToWaitFor_ = null; } + threadsToWaitForCase_ = 2; + onChanged(); return threadListBuilder_; } - private int policy_ = 0; + private java.util.List perThreadFailureHandlers_ = + java.util.Collections.emptyList(); + private void ensurePerThreadFailureHandlersIsMutable() { + if (!((bitField0_ & 0x00000004) != 0)) { + perThreadFailureHandlers_ = new java.util.ArrayList(perThreadFailureHandlers_); + bitField0_ |= 0x00000004; + } + } + + private com.google.protobuf.RepeatedFieldBuilderV3< + io.littlehorse.sdk.common.proto.FailureHandlerDef, io.littlehorse.sdk.common.proto.FailureHandlerDef.Builder, io.littlehorse.sdk.common.proto.FailureHandlerDefOrBuilder> perThreadFailureHandlersBuilder_; + /** - * .littlehorse.WaitForThreadsPolicy policy = 2; - * @return The enum numeric value on the wire for policy. + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; */ - @java.lang.Override public int getPolicyValue() { - return policy_; + public java.util.List getPerThreadFailureHandlersList() { + if (perThreadFailureHandlersBuilder_ == null) { + return java.util.Collections.unmodifiableList(perThreadFailureHandlers_); + } else { + return perThreadFailureHandlersBuilder_.getMessageList(); + } } /** - * .littlehorse.WaitForThreadsPolicy policy = 2; - * @param value The enum numeric value on the wire for policy to set. - * @return This builder for chaining. + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; */ - public Builder setPolicyValue(int value) { - policy_ = value; - bitField0_ |= 0x00000004; - onChanged(); + public int getPerThreadFailureHandlersCount() { + if (perThreadFailureHandlersBuilder_ == null) { + return perThreadFailureHandlers_.size(); + } else { + return perThreadFailureHandlersBuilder_.getCount(); + } + } + /** + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; + */ + public io.littlehorse.sdk.common.proto.FailureHandlerDef getPerThreadFailureHandlers(int index) { + if (perThreadFailureHandlersBuilder_ == null) { + return perThreadFailureHandlers_.get(index); + } else { + return perThreadFailureHandlersBuilder_.getMessage(index); + } + } + /** + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; + */ + public Builder setPerThreadFailureHandlers( + int index, io.littlehorse.sdk.common.proto.FailureHandlerDef value) { + if (perThreadFailureHandlersBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensurePerThreadFailureHandlersIsMutable(); + perThreadFailureHandlers_.set(index, value); + onChanged(); + } else { + perThreadFailureHandlersBuilder_.setMessage(index, value); + } return this; } /** - * .littlehorse.WaitForThreadsPolicy policy = 2; - * @return The policy. + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; */ - @java.lang.Override - public io.littlehorse.sdk.common.proto.WaitForThreadsPolicy getPolicy() { - io.littlehorse.sdk.common.proto.WaitForThreadsPolicy result = io.littlehorse.sdk.common.proto.WaitForThreadsPolicy.forNumber(policy_); - return result == null ? io.littlehorse.sdk.common.proto.WaitForThreadsPolicy.UNRECOGNIZED : result; + public Builder setPerThreadFailureHandlers( + int index, io.littlehorse.sdk.common.proto.FailureHandlerDef.Builder builderForValue) { + if (perThreadFailureHandlersBuilder_ == null) { + ensurePerThreadFailureHandlersIsMutable(); + perThreadFailureHandlers_.set(index, builderForValue.build()); + onChanged(); + } else { + perThreadFailureHandlersBuilder_.setMessage(index, builderForValue.build()); + } + return this; } /** - * .littlehorse.WaitForThreadsPolicy policy = 2; - * @param value The policy to set. - * @return This builder for chaining. + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; */ - public Builder setPolicy(io.littlehorse.sdk.common.proto.WaitForThreadsPolicy value) { - if (value == null) { - throw new NullPointerException(); + public Builder addPerThreadFailureHandlers(io.littlehorse.sdk.common.proto.FailureHandlerDef value) { + if (perThreadFailureHandlersBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensurePerThreadFailureHandlersIsMutable(); + perThreadFailureHandlers_.add(value); + onChanged(); + } else { + perThreadFailureHandlersBuilder_.addMessage(value); } - bitField0_ |= 0x00000004; - policy_ = value.getNumber(); - onChanged(); return this; } /** - * .littlehorse.WaitForThreadsPolicy policy = 2; - * @return This builder for chaining. + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; */ - public Builder clearPolicy() { - bitField0_ = (bitField0_ & ~0x00000004); - policy_ = 0; - onChanged(); + public Builder addPerThreadFailureHandlers( + int index, io.littlehorse.sdk.common.proto.FailureHandlerDef value) { + if (perThreadFailureHandlersBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensurePerThreadFailureHandlersIsMutable(); + perThreadFailureHandlers_.add(index, value); + onChanged(); + } else { + perThreadFailureHandlersBuilder_.addMessage(index, value); + } + return this; + } + /** + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; + */ + public Builder addPerThreadFailureHandlers( + io.littlehorse.sdk.common.proto.FailureHandlerDef.Builder builderForValue) { + if (perThreadFailureHandlersBuilder_ == null) { + ensurePerThreadFailureHandlersIsMutable(); + perThreadFailureHandlers_.add(builderForValue.build()); + onChanged(); + } else { + perThreadFailureHandlersBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; + */ + public Builder addPerThreadFailureHandlers( + int index, io.littlehorse.sdk.common.proto.FailureHandlerDef.Builder builderForValue) { + if (perThreadFailureHandlersBuilder_ == null) { + ensurePerThreadFailureHandlersIsMutable(); + perThreadFailureHandlers_.add(index, builderForValue.build()); + onChanged(); + } else { + perThreadFailureHandlersBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; + */ + public Builder addAllPerThreadFailureHandlers( + java.lang.Iterable values) { + if (perThreadFailureHandlersBuilder_ == null) { + ensurePerThreadFailureHandlersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll( + values, perThreadFailureHandlers_); + onChanged(); + } else { + perThreadFailureHandlersBuilder_.addAllMessages(values); + } + return this; + } + /** + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; + */ + public Builder clearPerThreadFailureHandlers() { + if (perThreadFailureHandlersBuilder_ == null) { + perThreadFailureHandlers_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000004); + onChanged(); + } else { + perThreadFailureHandlersBuilder_.clear(); + } + return this; + } + /** + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; + */ + public Builder removePerThreadFailureHandlers(int index) { + if (perThreadFailureHandlersBuilder_ == null) { + ensurePerThreadFailureHandlersIsMutable(); + perThreadFailureHandlers_.remove(index); + onChanged(); + } else { + perThreadFailureHandlersBuilder_.remove(index); + } return this; } + /** + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; + */ + public io.littlehorse.sdk.common.proto.FailureHandlerDef.Builder getPerThreadFailureHandlersBuilder( + int index) { + return getPerThreadFailureHandlersFieldBuilder().getBuilder(index); + } + /** + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; + */ + public io.littlehorse.sdk.common.proto.FailureHandlerDefOrBuilder getPerThreadFailureHandlersOrBuilder( + int index) { + if (perThreadFailureHandlersBuilder_ == null) { + return perThreadFailureHandlers_.get(index); } else { + return perThreadFailureHandlersBuilder_.getMessageOrBuilder(index); + } + } + /** + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; + */ + public java.util.List + getPerThreadFailureHandlersOrBuilderList() { + if (perThreadFailureHandlersBuilder_ != null) { + return perThreadFailureHandlersBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(perThreadFailureHandlers_); + } + } + /** + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; + */ + public io.littlehorse.sdk.common.proto.FailureHandlerDef.Builder addPerThreadFailureHandlersBuilder() { + return getPerThreadFailureHandlersFieldBuilder().addBuilder( + io.littlehorse.sdk.common.proto.FailureHandlerDef.getDefaultInstance()); + } + /** + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; + */ + public io.littlehorse.sdk.common.proto.FailureHandlerDef.Builder addPerThreadFailureHandlersBuilder( + int index) { + return getPerThreadFailureHandlersFieldBuilder().addBuilder( + index, io.littlehorse.sdk.common.proto.FailureHandlerDef.getDefaultInstance()); + } + /** + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; + */ + public java.util.List + getPerThreadFailureHandlersBuilderList() { + return getPerThreadFailureHandlersFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilderV3< + io.littlehorse.sdk.common.proto.FailureHandlerDef, io.littlehorse.sdk.common.proto.FailureHandlerDef.Builder, io.littlehorse.sdk.common.proto.FailureHandlerDefOrBuilder> + getPerThreadFailureHandlersFieldBuilder() { + if (perThreadFailureHandlersBuilder_ == null) { + perThreadFailureHandlersBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3< + io.littlehorse.sdk.common.proto.FailureHandlerDef, io.littlehorse.sdk.common.proto.FailureHandlerDef.Builder, io.littlehorse.sdk.common.proto.FailureHandlerDefOrBuilder>( + perThreadFailureHandlers_, + ((bitField0_ & 0x00000004) != 0), + getParentForChildren(), + isClean()); + perThreadFailureHandlers_ = null; + } + return perThreadFailureHandlersBuilder_; + } @java.lang.Override public final Builder setUnknownFields( final com.google.protobuf.UnknownFieldSet unknownFields) { diff --git a/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsNodeOrBuilder.java b/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsNodeOrBuilder.java index ee187884b..f5a65b2f0 100644 --- a/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsNodeOrBuilder.java +++ b/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsNodeOrBuilder.java @@ -8,77 +8,58 @@ public interface WaitForThreadsNodeOrBuilder extends com.google.protobuf.MessageOrBuilder { /** - * 
-   * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-   * for compatibility reasons, we cannot wrap it into a separate message.
-   *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.WaitForThreadsNode.ThreadsToWaitFor threads = 1; + * @return Whether the threads field is set. */ - java.util.List - getThreadsList(); + boolean hasThreads(); /** - * 
-   * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-   * for compatibility reasons, we cannot wrap it into a separate message.
-   *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.WaitForThreadsNode.ThreadsToWaitFor threads = 1; + * @return The threads. */ - io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor getThreads(int index); + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor getThreads(); /** - * 
-   * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-   * for compatibility reasons, we cannot wrap it into a separate message.
-   *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; + * .littlehorse.WaitForThreadsNode.ThreadsToWaitFor threads = 1; */ - int getThreadsCount(); - /** - * 
-   * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-   * for compatibility reasons, we cannot wrap it into a separate message.
-   *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; - */ - java.util.List - getThreadsOrBuilderList(); - /** - * 
-   * Either 1 or 3 is set. Cannot put `repeated` into a oneof, and
-   * for compatibility reasons, we cannot wrap it into a separate message.
-   *
- * - * repeated .littlehorse.WaitForThreadsNode.ThreadToWaitFor threads = 1; - */ - io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitForOrBuilder getThreadsOrBuilder( - int index); + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitForOrBuilder getThreadsOrBuilder(); /** - * optional .littlehorse.VariableAssignment thread_list = 3; + * .littlehorse.VariableAssignment thread_list = 2; * @return Whether the threadList field is set. */ boolean hasThreadList(); /** - * optional .littlehorse.VariableAssignment thread_list = 3; + * .littlehorse.VariableAssignment thread_list = 2; * @return The threadList. */ io.littlehorse.sdk.common.proto.VariableAssignment getThreadList(); /** - * optional .littlehorse.VariableAssignment thread_list = 3; + * .littlehorse.VariableAssignment thread_list = 2; */ io.littlehorse.sdk.common.proto.VariableAssignmentOrBuilder getThreadListOrBuilder(); /** - * .littlehorse.WaitForThreadsPolicy policy = 2; - * @return The enum numeric value on the wire for policy. + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; + */ + java.util.List + getPerThreadFailureHandlersList(); + /** + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; */ - int getPolicyValue(); + io.littlehorse.sdk.common.proto.FailureHandlerDef getPerThreadFailureHandlers(int index); /** - * .littlehorse.WaitForThreadsPolicy policy = 2; - * @return The policy. + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; */ - io.littlehorse.sdk.common.proto.WaitForThreadsPolicy getPolicy(); + int getPerThreadFailureHandlersCount(); + /** + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; + */ + java.util.List + getPerThreadFailureHandlersOrBuilderList(); + /** + * repeated .littlehorse.FailureHandlerDef per_thread_failure_handlers = 3; + */ + io.littlehorse.sdk.common.proto.FailureHandlerDefOrBuilder getPerThreadFailureHandlersOrBuilder( + int index); + + io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitForCase getThreadsToWaitForCase(); } diff --git a/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsPolicy.java b/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsPolicy.java deleted file mode 100644 index a383714b3..000000000 --- a/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsPolicy.java +++ /dev/null @@ -1,104 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: common_enums.proto - -package io.littlehorse.sdk.common.proto; - -/** - * Protobuf enum {@code littlehorse.WaitForThreadsPolicy} - */ -public enum WaitForThreadsPolicy - implements com.google.protobuf.ProtocolMessageEnum { - /** - * STOP_ON_FAILURE = 0; - */ - STOP_ON_FAILURE(0), - UNRECOGNIZED(-1), - ; - - /** - * STOP_ON_FAILURE = 0; - */ - public static final int STOP_ON_FAILURE_VALUE = 0; - - - public final int getNumber() { - if (this == UNRECOGNIZED) { - throw new java.lang.IllegalArgumentException( - "Can't get the number of an unknown enum value."); - } - return value; - } - - /** - * @param value The numeric wire value of the corresponding enum entry. - * @return The enum associated with the given numeric wire value. - * @deprecated Use {@link #forNumber(int)} instead. - */ - @java.lang.Deprecated - public static WaitForThreadsPolicy valueOf(int value) { - return forNumber(value); - } - - /** - * @param value The numeric wire value of the corresponding enum entry. - * @return The enum associated with the given numeric wire value. - */ - public static WaitForThreadsPolicy forNumber(int value) { - switch (value) { - case 0: return STOP_ON_FAILURE; - default: return null; - } - } - - public static com.google.protobuf.Internal.EnumLiteMap - internalGetValueMap() { - return internalValueMap; - } - private static final com.google.protobuf.Internal.EnumLiteMap< - WaitForThreadsPolicy> internalValueMap = - new com.google.protobuf.Internal.EnumLiteMap() { - public WaitForThreadsPolicy findValueByNumber(int number) { - return WaitForThreadsPolicy.forNumber(number); - } - }; - - public final com.google.protobuf.Descriptors.EnumValueDescriptor - getValueDescriptor() { - if (this == UNRECOGNIZED) { - throw new java.lang.IllegalStateException( - "Can't get the descriptor of an unrecognized enum value."); - } - return getDescriptor().getValues().get(ordinal()); - } - public final com.google.protobuf.Descriptors.EnumDescriptor - getDescriptorForType() { - return getDescriptor(); - } - public static final com.google.protobuf.Descriptors.EnumDescriptor - getDescriptor() { - return io.littlehorse.sdk.common.proto.CommonEnums.getDescriptor().getEnumTypes().get(6); - } - - private static final WaitForThreadsPolicy[] VALUES = values(); - - public static WaitForThreadsPolicy valueOf( - com.google.protobuf.Descriptors.EnumValueDescriptor desc) { - if (desc.getType() != getDescriptor()) { - throw new java.lang.IllegalArgumentException( - "EnumValueDescriptor is not for this type."); - } - if (desc.getIndex() == -1) { - return UNRECOGNIZED; - } - return VALUES[desc.getIndex()]; - } - - private final int value; - - private WaitForThreadsPolicy(int value) { - this.value = value; - } - - // @@protoc_insertion_point(enum_scope:littlehorse.WaitForThreadsPolicy) -} - diff --git a/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsRun.java b/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsRun.java index 0aec797ab..01200af2b 100644 --- a/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsRun.java +++ b/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsRun.java @@ -21,7 +21,6 @@ private WaitForThreadsRun(com.google.protobuf.GeneratedMessageV3.Builder buil } private WaitForThreadsRun() { threads_ = java.util.Collections.emptyList(); - policy_ = 0; } @java.lang.Override @@ -44,6 +43,176 @@ protected java.lang.Object newInstance( io.littlehorse.sdk.common.proto.WaitForThreadsRun.class, io.littlehorse.sdk.common.proto.WaitForThreadsRun.Builder.class); } + /** + * 
+   * The status of a single ThreadRun that we are waiting for.
+   *
+ * + * Protobuf enum {@code littlehorse.WaitForThreadsRun.WaitingThreadStatus} + */ + public enum WaitingThreadStatus + implements com.google.protobuf.ProtocolMessageEnum { + /** + * 
+     * The ThreadRun is in progress (i.e. not COMPLETED nor EXCEPTION nor ERROR)
+     *
+ * + * THREAD_IN_PROGRESS = 0; + */ + THREAD_IN_PROGRESS(0), + /** + * 
+     * The ThreadRun failed with some failure, and the FailureHandler is running
+     * for that Failure.
+     *
+ * + * THREAD_HANDLING_FAILURE = 1; + */ + THREAD_HANDLING_FAILURE(1), + /** + * 
+     * We can mark this ThreadRun as "already waited for", meaning that either:
+     * 1. It completed successfully, OR
+     * 2. It failed, and the Failure Handler successfully completed
+     *
+ * + * THREAD_COMPLETED_OR_FAILURE_HANDLED = 2; + */ + THREAD_COMPLETED_OR_FAILURE_HANDLED(2), + /** + * 
+     * The ThreadRun did not complete successfully, and there wasn't a successful
+     * run of a Failure Handler for the Failure that was thrown.
+     *
+ * + * THREAD_UNSUCCESSFUL = 3; + */ + THREAD_UNSUCCESSFUL(3), + UNRECOGNIZED(-1), + ; + + /** + * 
+     * The ThreadRun is in progress (i.e. not COMPLETED nor EXCEPTION nor ERROR)
+     *
+ * + * THREAD_IN_PROGRESS = 0; + */ + public static final int THREAD_IN_PROGRESS_VALUE = 0; + /** + * 
+     * The ThreadRun failed with some failure, and the FailureHandler is running
+     * for that Failure.
+     *
+ * + * THREAD_HANDLING_FAILURE = 1; + */ + public static final int THREAD_HANDLING_FAILURE_VALUE = 1; + /** + * 
+     * We can mark this ThreadRun as "already waited for", meaning that either:
+     * 1. It completed successfully, OR
+     * 2. It failed, and the Failure Handler successfully completed
+     *
+ * + * THREAD_COMPLETED_OR_FAILURE_HANDLED = 2; + */ + public static final int THREAD_COMPLETED_OR_FAILURE_HANDLED_VALUE = 2; + /** + * 
+     * The ThreadRun did not complete successfully, and there wasn't a successful
+     * run of a Failure Handler for the Failure that was thrown.
+     *
+ * + * THREAD_UNSUCCESSFUL = 3; + */ + public static final int THREAD_UNSUCCESSFUL_VALUE = 3; + + + public final int getNumber() { + if (this == UNRECOGNIZED) { + throw new java.lang.IllegalArgumentException( + "Can't get the number of an unknown enum value."); + } + return value; + } + + /** + * @param value The numeric wire value of the corresponding enum entry. + * @return The enum associated with the given numeric wire value. + * @deprecated Use {@link #forNumber(int)} instead. + */ + @java.lang.Deprecated + public static WaitingThreadStatus valueOf(int value) { + return forNumber(value); + } + + /** + * @param value The numeric wire value of the corresponding enum entry. + * @return The enum associated with the given numeric wire value. + */ + public static WaitingThreadStatus forNumber(int value) { + switch (value) { + case 0: return THREAD_IN_PROGRESS; + case 1: return THREAD_HANDLING_FAILURE; + case 2: return THREAD_COMPLETED_OR_FAILURE_HANDLED; + case 3: return THREAD_UNSUCCESSFUL; + default: return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap + internalGetValueMap() { + return internalValueMap; + } + private static final com.google.protobuf.Internal.EnumLiteMap< + WaitingThreadStatus> internalValueMap = + new com.google.protobuf.Internal.EnumLiteMap() { + public WaitingThreadStatus findValueByNumber(int number) { + return WaitingThreadStatus.forNumber(number); + } + }; + + public final com.google.protobuf.Descriptors.EnumValueDescriptor + getValueDescriptor() { + if (this == UNRECOGNIZED) { + throw new java.lang.IllegalStateException( + "Can't get the descriptor of an unrecognized enum value."); + } + return getDescriptor().getValues().get(ordinal()); + } + public final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptorForType() { + return getDescriptor(); + } + public static final com.google.protobuf.Descriptors.EnumDescriptor + getDescriptor() { + return io.littlehorse.sdk.common.proto.WaitForThreadsRun.getDescriptor().getEnumTypes().get(0); + } + + private static final WaitingThreadStatus[] VALUES = values(); + + public static WaitingThreadStatus valueOf( + com.google.protobuf.Descriptors.EnumValueDescriptor desc) { + if (desc.getType() != getDescriptor()) { + throw new java.lang.IllegalArgumentException( + "EnumValueDescriptor is not for this type."); + } + if (desc.getIndex() == -1) { + return UNRECOGNIZED; + } + return VALUES[desc.getIndex()]; + } + + private final int value; + + private WaitingThreadStatus(int value) { + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:littlehorse.WaitForThreadsRun.WaitingThreadStatus) + } + public interface WaitForThreadOrBuilder extends // @@protoc_insertion_point(interface_extends:littlehorse.WaitForThreadsRun.WaitForThread) com.google.protobuf.MessageOrBuilder { @@ -109,13 +278,47 @@ public interface WaitForThreadOrBuilder extends /** * 
-     * INTERNAL: flag used by scheduler internally.
+     * The "waiting status" of this specific thread: whether it's still running,
+     * already done, handling a failure, or completely failed.
+     *
+ * + * .littlehorse.WaitForThreadsRun.WaitingThreadStatus waiting_status = 4; + * @return The enum numeric value on the wire for waitingStatus. + */ + int getWaitingStatusValue(); + /** + * 
+     * The "waiting status" of this specific thread: whether it's still running,
+     * already done, handling a failure, or completely failed.
+     *
+ * + * .littlehorse.WaitForThreadsRun.WaitingThreadStatus waiting_status = 4; + * @return The waitingStatus. + */ + io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitingThreadStatus getWaitingStatus(); + + /** + * 
+     * If there is a failure on the ThreadRun, and we have a failure handler defined
+     * for it, then we will start a failure handler for this threadrun. This field
+     * is the id of that threadRun.
      *
* - * bool already_handled = 5; - * @return The alreadyHandled. + * optional int32 failure_handler_thread_run_id = 5; + * @return Whether the failureHandlerThreadRunId field is set. */ - boolean getAlreadyHandled(); + boolean hasFailureHandlerThreadRunId(); + /** + * 
+     * If there is a failure on the ThreadRun, and we have a failure handler defined
+     * for it, then we will start a failure handler for this threadrun. This field
+     * is the id of that threadRun.
+     *
+ * + * optional int32 failure_handler_thread_run_id = 5; + * @return The failureHandlerThreadRunId. + */ + int getFailureHandlerThreadRunId(); } /** * 
@@ -135,6 +338,7 @@ private WaitForThread(com.google.protobuf.GeneratedMessageV3.Builder builder)
     }
     private WaitForThread() {
       threadStatus_ = 0;
+      waitingStatus_ = 0;
     }
 
     @java.lang.Override
@@ -240,19 +444,63 @@ public int getThreadRunNumber() {
       return threadRunNumber_;
     }
 
-    public static final int ALREADY_HANDLED_FIELD_NUMBER = 5;
-    private boolean alreadyHandled_ = false;
+    public static final int WAITING_STATUS_FIELD_NUMBER = 4;
+    private int waitingStatus_ = 0;
     /**
      * 
-     * INTERNAL: flag used by scheduler internally.
+     * The "waiting status" of this specific thread: whether it's still running,
+     * already done, handling a failure, or completely failed.
      * 

      *
-     * bool already_handled = 5;
-     * @return The alreadyHandled.
+     * .littlehorse.WaitForThreadsRun.WaitingThreadStatus waiting_status = 4;
+     * @return The enum numeric value on the wire for waitingStatus.
+     */
+    @java.lang.Override public int getWaitingStatusValue() {
+      return waitingStatus_;
+    }
+    /**
+     * 
+     * The "waiting status" of this specific thread: whether it's still running,
+     * already done, handling a failure, or completely failed.
+     * 

+     *
+     * .littlehorse.WaitForThreadsRun.WaitingThreadStatus waiting_status = 4;
+     * @return The waitingStatus.
+     */
+    @java.lang.Override public io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitingThreadStatus getWaitingStatus() {
+      io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitingThreadStatus result = io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitingThreadStatus.forNumber(waitingStatus_);
+      return result == null ? io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitingThreadStatus.UNRECOGNIZED : result;
+    }
+
+    public static final int FAILURE_HANDLER_THREAD_RUN_ID_FIELD_NUMBER = 5;
+    private int failureHandlerThreadRunId_ = 0;
+    /**
+     * 
+     * If there is a failure on the ThreadRun, and we have a failure handler defined
+     * for it, then we will start a failure handler for this threadrun. This field
+     * is the id of that threadRun.
+     * 

+     *
+     * optional int32 failure_handler_thread_run_id = 5;
+     * @return Whether the failureHandlerThreadRunId field is set.
+     */
+    @java.lang.Override
+    public boolean hasFailureHandlerThreadRunId() {
+      return ((bitField0_ & 0x00000002) != 0);
+    }
+    /**
+     * 
+     * If there is a failure on the ThreadRun, and we have a failure handler defined
+     * for it, then we will start a failure handler for this threadrun. This field
+     * is the id of that threadRun.
+     * 

+     *
+     * optional int32 failure_handler_thread_run_id = 5;
+     * @return The failureHandlerThreadRunId.
      */
     @java.lang.Override
-    public boolean getAlreadyHandled() {
-      return alreadyHandled_;
+    public int getFailureHandlerThreadRunId() {
+      return failureHandlerThreadRunId_;
     }
 
     private byte memoizedIsInitialized = -1;
@@ -278,8 +526,11 @@ public void writeTo(com.google.protobuf.CodedOutputStream output)
       if (threadRunNumber_ != 0) {
         output.writeInt32(3, threadRunNumber_);
       }
-      if (alreadyHandled_ != false) {
-        output.writeBool(5, alreadyHandled_);
+      if (waitingStatus_ != io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitingThreadStatus.THREAD_IN_PROGRESS.getNumber()) {
+        output.writeEnum(4, waitingStatus_);
+      }
+      if (((bitField0_ & 0x00000002) != 0)) {
+        output.writeInt32(5, failureHandlerThreadRunId_);
       }
       getUnknownFields().writeTo(output);
     }
@@ -302,9 +553,13 @@ public int getSerializedSize() {
         size += com.google.protobuf.CodedOutputStream
           .computeInt32Size(3, threadRunNumber_);
       }
-      if (alreadyHandled_ != false) {
+      if (waitingStatus_ != io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitingThreadStatus.THREAD_IN_PROGRESS.getNumber()) {
         size += com.google.protobuf.CodedOutputStream
-          .computeBoolSize(5, alreadyHandled_);
+          .computeEnumSize(4, waitingStatus_);
+      }
+      if (((bitField0_ & 0x00000002) != 0)) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeInt32Size(5, failureHandlerThreadRunId_);
       }
       size += getUnknownFields().getSerializedSize();
       memoizedSize = size;
@@ -329,8 +584,12 @@ public boolean equals(final java.lang.Object obj) {
       if (threadStatus_ != other.threadStatus_) return false;
       if (getThreadRunNumber()
           != other.getThreadRunNumber()) return false;
-      if (getAlreadyHandled()
-          != other.getAlreadyHandled()) return false;
+      if (waitingStatus_ != other.waitingStatus_) return false;
+      if (hasFailureHandlerThreadRunId() != other.hasFailureHandlerThreadRunId()) return false;
+      if (hasFailureHandlerThreadRunId()) {
+        if (getFailureHandlerThreadRunId()
+            != other.getFailureHandlerThreadRunId()) return false;
+      }
       if (!getUnknownFields().equals(other.getUnknownFields())) return false;
       return true;
     }
@@ -350,9 +609,12 @@ public int hashCode() {
       hash = (53 * hash) + threadStatus_;
       hash = (37 * hash) + THREAD_RUN_NUMBER_FIELD_NUMBER;
       hash = (53 * hash) + getThreadRunNumber();
-      hash = (37 * hash) + ALREADY_HANDLED_FIELD_NUMBER;
-      hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(
-          getAlreadyHandled());
+      hash = (37 * hash) + WAITING_STATUS_FIELD_NUMBER;
+      hash = (53 * hash) + waitingStatus_;
+      if (hasFailureHandlerThreadRunId()) {
+        hash = (37 * hash) + FAILURE_HANDLER_THREAD_RUN_ID_FIELD_NUMBER;
+        hash = (53 * hash) + getFailureHandlerThreadRunId();
+      }
       hash = (29 * hash) + getUnknownFields().hashCode();
       memoizedHashCode = hash;
       return hash;
@@ -501,7 +763,8 @@ public Builder clear() {
         }
         threadStatus_ = 0;
         threadRunNumber_ = 0;
-        alreadyHandled_ = false;
+        waitingStatus_ = 0;
+        failureHandlerThreadRunId_ = 0;
         return this;
       }
 
@@ -549,7 +812,11 @@ private void buildPartial0(io.littlehorse.sdk.common.proto.WaitForThreadsRun.Wai
           result.threadRunNumber_ = threadRunNumber_;
         }
         if (((from_bitField0_ & 0x00000008) != 0)) {
-          result.alreadyHandled_ = alreadyHandled_;
+          result.waitingStatus_ = waitingStatus_;
+        }
+        if (((from_bitField0_ & 0x00000010) != 0)) {
+          result.failureHandlerThreadRunId_ = failureHandlerThreadRunId_;
+          to_bitField0_ |= 0x00000002;
         }
         result.bitField0_ |= to_bitField0_;
       }
@@ -607,8 +874,11 @@ public Builder mergeFrom(io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitF
         if (other.getThreadRunNumber() != 0) {
           setThreadRunNumber(other.getThreadRunNumber());
         }
-        if (other.getAlreadyHandled() != false) {
-          setAlreadyHandled(other.getAlreadyHandled());
+        if (other.waitingStatus_ != 0) {
+          setWaitingStatusValue(other.getWaitingStatusValue());
+        }
+        if (other.hasFailureHandlerThreadRunId()) {
+          setFailureHandlerThreadRunId(other.getFailureHandlerThreadRunId());
         }
         this.mergeUnknownFields(other.getUnknownFields());
         onChanged();
@@ -653,10 +923,15 @@ public Builder mergeFrom(
                 bitField0_ |= 0x00000004;
                 break;
               } // case 24
-              case 40: {
-                alreadyHandled_ = input.readBool();
+              case 32: {
+                waitingStatus_ = input.readEnum();
                 bitField0_ |= 0x00000008;
                 break;
+              } // case 32
+              case 40: {
+                failureHandlerThreadRunId_ = input.readInt32();
+                bitField0_ |= 0x00000010;
+                break;
               } // case 40
               default: {
                 if (!super.parseUnknownField(input, extensionRegistry, tag)) {
@@ -956,46 +1231,144 @@ public Builder clearThreadRunNumber() {
         return this;
       }
 
-      private boolean alreadyHandled_ ;
+      private int waitingStatus_ = 0;
+      /**
+       * 
+       * The "waiting status" of this specific thread: whether it's still running,
+       * already done, handling a failure, or completely failed.
+       * 

+       *
+       * .littlehorse.WaitForThreadsRun.WaitingThreadStatus waiting_status = 4;
+       * @return The enum numeric value on the wire for waitingStatus.
+       */
+      @java.lang.Override public int getWaitingStatusValue() {
+        return waitingStatus_;
+      }
+      /**
+       * 
+       * The "waiting status" of this specific thread: whether it's still running,
+       * already done, handling a failure, or completely failed.
+       * 

+       *
+       * .littlehorse.WaitForThreadsRun.WaitingThreadStatus waiting_status = 4;
+       * @param value The enum numeric value on the wire for waitingStatus to set.
+       * @return This builder for chaining.
+       */
+      public Builder setWaitingStatusValue(int value) {
+        waitingStatus_ = value;
+        bitField0_ |= 0x00000008;
+        onChanged();
+        return this;
+      }
       /**
        * 
-       * INTERNAL: flag used by scheduler internally.
+       * The "waiting status" of this specific thread: whether it's still running,
+       * already done, handling a failure, or completely failed.
        * 

        *
-       * bool already_handled = 5;
-       * @return The alreadyHandled.
+       * .littlehorse.WaitForThreadsRun.WaitingThreadStatus waiting_status = 4;
+       * @return The waitingStatus.
        */
       @java.lang.Override
-      public boolean getAlreadyHandled() {
-        return alreadyHandled_;
+      public io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitingThreadStatus getWaitingStatus() {
+        io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitingThreadStatus result = io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitingThreadStatus.forNumber(waitingStatus_);
+        return result == null ? io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitingThreadStatus.UNRECOGNIZED : result;
       }
       /**
        * 
-       * INTERNAL: flag used by scheduler internally.
+       * The "waiting status" of this specific thread: whether it's still running,
+       * already done, handling a failure, or completely failed.
        * 

        *
-       * bool already_handled = 5;
-       * @param value The alreadyHandled to set.
+       * .littlehorse.WaitForThreadsRun.WaitingThreadStatus waiting_status = 4;
+       * @param value The waitingStatus to set.
        * @return This builder for chaining.
        */
-      public Builder setAlreadyHandled(boolean value) {
-
-        alreadyHandled_ = value;
+      public Builder setWaitingStatus(io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitingThreadStatus value) {
+        if (value == null) {
+          throw new NullPointerException();
+        }
         bitField0_ |= 0x00000008;
+        waitingStatus_ = value.getNumber();
         onChanged();
         return this;
       }
       /**
        * 
-       * INTERNAL: flag used by scheduler internally.
+       * The "waiting status" of this specific thread: whether it's still running,
+       * already done, handling a failure, or completely failed.
        * 

        *
-       * bool already_handled = 5;
+       * .littlehorse.WaitForThreadsRun.WaitingThreadStatus waiting_status = 4;
        * @return This builder for chaining.
        */
-      public Builder clearAlreadyHandled() {
+      public Builder clearWaitingStatus() {
         bitField0_ = (bitField0_ & ~0x00000008);
-        alreadyHandled_ = false;
+        waitingStatus_ = 0;
+        onChanged();
+        return this;
+      }
+
+      private int failureHandlerThreadRunId_ ;
+      /**
+       * 
+       * If there is a failure on the ThreadRun, and we have a failure handler defined
+       * for it, then we will start a failure handler for this threadrun. This field
+       * is the id of that threadRun.
+       * 

+       *
+       * optional int32 failure_handler_thread_run_id = 5;
+       * @return Whether the failureHandlerThreadRunId field is set.
+       */
+      @java.lang.Override
+      public boolean hasFailureHandlerThreadRunId() {
+        return ((bitField0_ & 0x00000010) != 0);
+      }
+      /**
+       * 
+       * If there is a failure on the ThreadRun, and we have a failure handler defined
+       * for it, then we will start a failure handler for this threadrun. This field
+       * is the id of that threadRun.
+       * 

+       *
+       * optional int32 failure_handler_thread_run_id = 5;
+       * @return The failureHandlerThreadRunId.
+       */
+      @java.lang.Override
+      public int getFailureHandlerThreadRunId() {
+        return failureHandlerThreadRunId_;
+      }
+      /**
+       * 
+       * If there is a failure on the ThreadRun, and we have a failure handler defined
+       * for it, then we will start a failure handler for this threadrun. This field
+       * is the id of that threadRun.
+       * 

+       *
+       * optional int32 failure_handler_thread_run_id = 5;
+       * @param value The failureHandlerThreadRunId to set.
+       * @return This builder for chaining.
+       */
+      public Builder setFailureHandlerThreadRunId(int value) {
+
+        failureHandlerThreadRunId_ = value;
+        bitField0_ |= 0x00000010;
+        onChanged();
+        return this;
+      }
+      /**
+       * 
+       * If there is a failure on the ThreadRun, and we have a failure handler defined
+       * for it, then we will start a failure handler for this threadrun. This field
+       * is the id of that threadRun.
+       * 

+       *
+       * optional int32 failure_handler_thread_run_id = 5;
+       * @return This builder for chaining.
+       */
+      public Builder clearFailureHandlerThreadRunId() {
+        bitField0_ = (bitField0_ & ~0x00000010);
+        failureHandlerThreadRunId_ = 0;
         onChanged();
         return this;
       }
@@ -1124,34 +1497,6 @@ public io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitForThreadOrBuilder
     return threads_.get(index);
   }
 
-  public static final int POLICY_FIELD_NUMBER = 2;
-  private int policy_ = 0;
-  /**
-   * 
-   * The policy to use when handling failures for Threads. Currently, only
-   * one policy exists.
-   * 

-   *
-   * .littlehorse.WaitForThreadsPolicy policy = 2;
-   * @return The enum numeric value on the wire for policy.
-   */
-  @java.lang.Override public int getPolicyValue() {
-    return policy_;
-  }
-  /**
-   * 
-   * The policy to use when handling failures for Threads. Currently, only
-   * one policy exists.
-   * 

-   *
-   * .littlehorse.WaitForThreadsPolicy policy = 2;
-   * @return The policy.
-   */
-  @java.lang.Override public io.littlehorse.sdk.common.proto.WaitForThreadsPolicy getPolicy() {
-    io.littlehorse.sdk.common.proto.WaitForThreadsPolicy result = io.littlehorse.sdk.common.proto.WaitForThreadsPolicy.forNumber(policy_);
-    return result == null ? io.littlehorse.sdk.common.proto.WaitForThreadsPolicy.UNRECOGNIZED : result;
-  }
-
   private byte memoizedIsInitialized = -1;
   @java.lang.Override
   public final boolean isInitialized() {
@@ -1169,9 +1514,6 @@ public void writeTo(com.google.protobuf.CodedOutputStream output)
     for (int i = 0; i < threads_.size(); i++) {
       output.writeMessage(1, threads_.get(i));
     }
-    if (policy_ != io.littlehorse.sdk.common.proto.WaitForThreadsPolicy.STOP_ON_FAILURE.getNumber()) {
-      output.writeEnum(2, policy_);
-    }
     getUnknownFields().writeTo(output);
   }
 
@@ -1185,10 +1527,6 @@ public int getSerializedSize() {
       size += com.google.protobuf.CodedOutputStream
         .computeMessageSize(1, threads_.get(i));
     }
-    if (policy_ != io.littlehorse.sdk.common.proto.WaitForThreadsPolicy.STOP_ON_FAILURE.getNumber()) {
-      size += com.google.protobuf.CodedOutputStream
-        .computeEnumSize(2, policy_);
-    }
     size += getUnknownFields().getSerializedSize();
     memoizedSize = size;
     return size;
@@ -1206,7 +1544,6 @@ public boolean equals(final java.lang.Object obj) {
 
     if (!getThreadsList()
         .equals(other.getThreadsList())) return false;
-    if (policy_ != other.policy_) return false;
     if (!getUnknownFields().equals(other.getUnknownFields())) return false;
     return true;
   }
@@ -1222,8 +1559,6 @@ public int hashCode() {
       hash = (37 * hash) + THREADS_FIELD_NUMBER;
       hash = (53 * hash) + getThreadsList().hashCode();
     }
-    hash = (37 * hash) + POLICY_FIELD_NUMBER;
-    hash = (53 * hash) + policy_;
     hash = (29 * hash) + getUnknownFields().hashCode();
     memoizedHashCode = hash;
     return hash;
@@ -1366,7 +1701,6 @@ public Builder clear() {
         threadsBuilder_.clear();
       }
       bitField0_ = (bitField0_ & ~0x00000001);
-      policy_ = 0;
       return this;
     }
 
@@ -1413,9 +1747,6 @@ private void buildPartialRepeatedFields(io.littlehorse.sdk.common.proto.WaitForT
 
     private void buildPartial0(io.littlehorse.sdk.common.proto.WaitForThreadsRun result) {
       int from_bitField0_ = bitField0_;
-      if (((from_bitField0_ & 0x00000002) != 0)) {
-        result.policy_ = policy_;
-      }
     }
 
     @java.lang.Override
@@ -1488,9 +1819,6 @@ public Builder mergeFrom(io.littlehorse.sdk.common.proto.WaitForThreadsRun other
           }
         }
       }
-      if (other.policy_ != 0) {
-        setPolicyValue(other.getPolicyValue());
-      }
       this.mergeUnknownFields(other.getUnknownFields());
       onChanged();
       return this;
@@ -1530,11 +1858,6 @@ public Builder mergeFrom(
               }
               break;
             } // case 10
-            case 16: {
-              policy_ = input.readEnum();
-              bitField0_ |= 0x00000002;
-              break;
-            } // case 16
             default: {
               if (!super.parseUnknownField(input, extensionRegistry, tag)) {
                 done = true; // was an endgroup tag
@@ -1863,84 +2186,6 @@ public io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitForThread.Builder a
       }
       return threadsBuilder_;
     }
-
-    private int policy_ = 0;
-    /**
-     * 
-     * The policy to use when handling failures for Threads. Currently, only
-     * one policy exists.
-     * 

-     *
-     * .littlehorse.WaitForThreadsPolicy policy = 2;
-     * @return The enum numeric value on the wire for policy.
-     */
-    @java.lang.Override public int getPolicyValue() {
-      return policy_;
-    }
-    /**
-     * 
-     * The policy to use when handling failures for Threads. Currently, only
-     * one policy exists.
-     * 

-     *
-     * .littlehorse.WaitForThreadsPolicy policy = 2;
-     * @param value The enum numeric value on the wire for policy to set.
-     * @return This builder for chaining.
-     */
-    public Builder setPolicyValue(int value) {
-      policy_ = value;
-      bitField0_ |= 0x00000002;
-      onChanged();
-      return this;
-    }
-    /**
-     * 
-     * The policy to use when handling failures for Threads. Currently, only
-     * one policy exists.
-     * 

-     *
-     * .littlehorse.WaitForThreadsPolicy policy = 2;
-     * @return The policy.
-     */
-    @java.lang.Override
-    public io.littlehorse.sdk.common.proto.WaitForThreadsPolicy getPolicy() {
-      io.littlehorse.sdk.common.proto.WaitForThreadsPolicy result = io.littlehorse.sdk.common.proto.WaitForThreadsPolicy.forNumber(policy_);
-      return result == null ? io.littlehorse.sdk.common.proto.WaitForThreadsPolicy.UNRECOGNIZED : result;
-    }
-    /**
-     * 
-     * The policy to use when handling failures for Threads. Currently, only
-     * one policy exists.
-     * 

-     *
-     * .littlehorse.WaitForThreadsPolicy policy = 2;
-     * @param value The policy to set.
-     * @return This builder for chaining.
-     */
-    public Builder setPolicy(io.littlehorse.sdk.common.proto.WaitForThreadsPolicy value) {
-      if (value == null) {
-        throw new NullPointerException();
-      }
-      bitField0_ |= 0x00000002;
-      policy_ = value.getNumber();
-      onChanged();
-      return this;
-    }
-    /**
-     * 
-     * The policy to use when handling failures for Threads. Currently, only
-     * one policy exists.
-     * 

-     *
-     * .littlehorse.WaitForThreadsPolicy policy = 2;
-     * @return This builder for chaining.
-     */
-    public Builder clearPolicy() {
-      bitField0_ = (bitField0_ & ~0x00000002);
-      policy_ = 0;
-      onChanged();
-      return this;
-    }
     @java.lang.Override
     public final Builder setUnknownFields(
         final com.google.protobuf.UnknownFieldSet unknownFields) {
diff --git a/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsRunOrBuilder.java b/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsRunOrBuilder.java
index ebed4c143..fa21ea849 100644
--- a/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsRunOrBuilder.java
+++ b/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WaitForThreadsRunOrBuilder.java
@@ -50,25 +50,4 @@ public interface WaitForThreadsRunOrBuilder extends
    */
   io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitForThreadOrBuilder getThreadsOrBuilder(
       int index);
-
-  /**
-   * 
-   * The policy to use when handling failures for Threads. Currently, only
-   * one policy exists.
-   * 

-   *
-   * .littlehorse.WaitForThreadsPolicy policy = 2;
-   * @return The enum numeric value on the wire for policy.
-   */
-  int getPolicyValue();
-  /**
-   * 
-   * The policy to use when handling failures for Threads. Currently, only
-   * one policy exists.
-   * 

-   *
-   * .littlehorse.WaitForThreadsPolicy policy = 2;
-   * @return The policy.
-   */
-  io.littlehorse.sdk.common.proto.WaitForThreadsPolicy getPolicy();
 }
diff --git a/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WfSpecOuterClass.java b/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WfSpecOuterClass.java
index 7d6b3b004..c582741b8 100644
--- a/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WfSpecOuterClass.java
+++ b/sdk-java/src/main/java/io/littlehorse/sdk/common/proto/WfSpecOuterClass.java
@@ -104,6 +104,11 @@ public static void registerAllExtensions(
   static final 
     com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
       internal_static_littlehorse_WaitForThreadsNode_ThreadToWaitFor_fieldAccessorTable;
+  static final com.google.protobuf.Descriptors.Descriptor
+    internal_static_littlehorse_WaitForThreadsNode_ThreadsToWaitFor_descriptor;
+  static final 
+    com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+      internal_static_littlehorse_WaitForThreadsNode_ThreadsToWaitFor_fieldAccessorTable;
   static final com.google.protobuf.Descriptors.Descriptor
     internal_static_littlehorse_ExternalEventNode_descriptor;
   static final 
@@ -249,78 +254,81 @@ public static void registerAllExtensions(
       "e\030\003 \001(\0162,.littlehorse.FailureHandlerDef." +
       "LHFailureTypeH\000\"C\n\rLHFailureType\022\026\n\022FAIL" +
       "URE_TYPE_ERROR\020\000\022\032\n\026FAILURE_TYPE_EXCEPTI" +
-      "ON\020\001B\022\n\020failure_to_catch\"\243\002\n\022WaitForThre" +
-      "adsNode\022@\n\007threads\030\001 \003(\0132/.littlehorse.W" +
-      "aitForThreadsNode.ThreadToWaitFor\0229\n\013thr" +
-      "ead_list\030\003 \001(\0132\037.littlehorse.VariableAss" +
-      "ignmentH\000\210\001\001\0221\n\006policy\030\002 \001(\0162!.littlehor" +
-      "se.WaitForThreadsPolicy\032M\n\017ThreadToWaitF" +
-      "or\022:\n\021thread_run_number\030\001 \001(\0132\037.littleho" +
-      "rse.VariableAssignmentB\016\n\014_thread_list\"\215" +
-      "\001\n\021ExternalEventNode\022>\n\025external_event_d" +
-      "ef_id\030\001 \001(\0132\037.littlehorse.ExternalEventD" +
-      "efId\0228\n\017timeout_seconds\030\002 \001(\0132\037.littleho" +
-      "rse.VariableAssignment\"\020\n\016EntrypointNode" +
-      "\"M\n\010ExitNode\0221\n\013failure_def\030\001 \001(\0132\027.litt" +
-      "lehorse.FailureDefH\000\210\001\001B\016\n\014_failure_def\"" +
-      "v\n\nFailureDef\022\024\n\014failure_name\030\001 \001(\t\022\017\n\007m" +
-      "essage\030\002 \001(\t\0225\n\007content\030\003 \001(\0132\037.littleho" +
-      "rse.VariableAssignmentH\000\210\001\001B\n\n\010_content\"" +
-      "\350\004\n\004Node\022)\n\016outgoing_edges\030\001 \003(\0132\021.littl" +
-      "ehorse.Edge\0228\n\020failure_handlers\030\004 \003(\0132\036." +
-      "littlehorse.FailureHandlerDef\0221\n\nentrypo" +
-      "int\030\005 \001(\0132\033.littlehorse.EntrypointNodeH\000" +
-      "\022%\n\004exit\030\006 \001(\0132\025.littlehorse.ExitNodeH\000\022" +
-      "%\n\004task\030\007 \001(\0132\025.littlehorse.TaskNodeH\000\0228" +
-      "\n\016external_event\030\010 \001(\0132\036.littlehorse.Ext" +
-      "ernalEventNodeH\000\0224\n\014start_thread\030\t \001(\0132\034" +
-      ".littlehorse.StartThreadNodeH\000\022;\n\020wait_f" +
-      "or_threads\030\n \001(\0132\037.littlehorse.WaitForTh" +
-      "readsNodeH\000\022#\n\003nop\030\013 \001(\0132\024.littlehorse.N" +
-      "opNodeH\000\022\'\n\005sleep\030\014 \001(\0132\026.littlehorse.Sl" +
-      "eepNodeH\000\022.\n\tuser_task\030\r \001(\0132\031.littlehor" +
-      "se.UserTaskNodeH\000\022G\n\026start_multiple_thre" +
-      "ads\030\017 \001(\0132%.littlehorse.StartMultipleThr" +
-      "eadsNodeH\000B\006\n\004node\"\342\002\n\014UserTaskNode\022\032\n\022u" +
-      "ser_task_def_name\030\001 \001(\t\0228\n\nuser_group\030\002 " +
-      "\001(\0132\037.littlehorse.VariableAssignmentH\000\210\001" +
-      "\001\0225\n\007user_id\030\003 \001(\0132\037.littlehorse.Variabl" +
-      "eAssignmentH\001\210\001\001\022-\n\007actions\030\004 \003(\0132\034.litt" +
-      "lehorse.UTActionTrigger\022\"\n\025user_task_def" +
-      "_version\030\005 \001(\005H\002\210\001\001\0223\n\005notes\030\006 \001(\0132\037.lit" +
-      "tlehorse.VariableAssignmentH\003\210\001\001B\r\n\013_use" +
-      "r_groupB\n\n\010_user_idB\030\n\026_user_task_def_ve" +
-      "rsionB\010\n\006_notes\"\233\001\n\rEdgeCondition\022+\n\ncom" +
-      "parator\030\001 \001(\0162\027.littlehorse.Comparator\022-" +
-      "\n\004left\030\002 \001(\0132\037.littlehorse.VariableAssig" +
-      "nment\022.\n\005right\030\003 \001(\0132\037.littlehorse.Varia" +
-      "bleAssignment\"\233\001\n\004Edge\022\026\n\016sink_node_name" +
-      "\030\001 \001(\t\0222\n\tcondition\030\002 \001(\0132\032.littlehorse." +
-      "EdgeConditionH\000\210\001\001\0229\n\022variable_mutations" +
-      "\030\003 \003(\0132\035.littlehorse.VariableMutationB\014\n" +
-      "\n_condition\"\t\n\007NopNode\"\276\001\n\tSleepNode\0226\n\013" +
-      "raw_seconds\030\001 \001(\0132\037.littlehorse.Variable" +
-      "AssignmentH\000\0224\n\ttimestamp\030\002 \001(\0132\037.little" +
-      "horse.VariableAssignmentH\000\0223\n\010iso_date\030\003" +
-      " \001(\0132\037.littlehorse.VariableAssignmentH\000B" +
-      "\016\n\014sleep_length\"\207\002\n\026WfSpecVersionMigrati" +
-      "on\022\031\n\021new_major_version\030\001 \001(\005\022\024\n\014new_rev" +
-      "ision\030\002 \001(\005\022]\n\026thread_spec_migrations\030\003 " +
-      "\003(\0132=.littlehorse.WfSpecVersionMigration" +
-      ".ThreadSpecMigrationsEntry\032]\n\031ThreadSpec" +
-      "MigrationsEntry\022\013\n\003key\030\001 \001(\t\022/\n\005value\030\002 " +
-      "\001(\0132 .littlehorse.ThreadSpecMigration:\0028" +
-      "\001\"\325\001\n\023ThreadSpecMigration\022\034\n\024new_thread_" +
-      "spec_name\030\001 \001(\t\022M\n\017node_migrations\030\002 \003(\013" +
-      "24.littlehorse.ThreadSpecMigration.NodeM" +
-      "igrationsEntry\032Q\n\023NodeMigrationsEntry\022\013\n" +
-      "\003key\030\001 \001(\t\022)\n\005value\030\002 \001(\0132\032.littlehorse." +
-      "NodeMigration:\0028\001\"&\n\rNodeMigration\022\025\n\rne" +
-      "w_node_name\030\001 \001(\t*N\n\030WfRunVariableAccess" +
-      "Level\022\016\n\nPUBLIC_VAR\020\000\022\017\n\013PRIVATE_VAR\020\001\022\021" +
-      "\n\rINHERITED_VAR\020\002BG\n\037io.littlehorse.sdk." +
-      "common.protoP\001Z\007.;model\252\002\030LittleHorse.Co" +
-      "mmon.Protob\006proto3"
+      "ON\020\001B\022\n\020failure_to_catch\"\222\003\n\022WaitForThre" +
+      "adsNode\022C\n\007threads\030\001 \001(\01320.littlehorse.W" +
+      "aitForThreadsNode.ThreadsToWaitForH\000\0226\n\013" +
+      "thread_list\030\002 \001(\0132\037.littlehorse.Variable" +
+      "AssignmentH\000\022C\n\033per_thread_failure_handl" +
+      "ers\030\003 \003(\0132\036.littlehorse.FailureHandlerDe" +
+      "f\032M\n\017ThreadToWaitFor\022:\n\021thread_run_numbe" +
+      "r\030\001 \001(\0132\037.littlehorse.VariableAssignment" +
+      "\032T\n\020ThreadsToWaitFor\022@\n\007threads\030\001 \003(\0132/." +
+      "littlehorse.WaitForThreadsNode.ThreadToW" +
+      "aitForB\025\n\023threads_to_wait_for\"\215\001\n\021Extern" +
+      "alEventNode\022>\n\025external_event_def_id\030\001 \001" +
+      "(\0132\037.littlehorse.ExternalEventDefId\0228\n\017t" +
+      "imeout_seconds\030\002 \001(\0132\037.littlehorse.Varia" +
+      "bleAssignment\"\020\n\016EntrypointNode\"M\n\010ExitN" +
+      "ode\0221\n\013failure_def\030\001 \001(\0132\027.littlehorse.F" +
+      "ailureDefH\000\210\001\001B\016\n\014_failure_def\"v\n\nFailur" +
+      "eDef\022\024\n\014failure_name\030\001 \001(\t\022\017\n\007message\030\002 " +
+      "\001(\t\0225\n\007content\030\003 \001(\0132\037.littlehorse.Varia" +
+      "bleAssignmentH\000\210\001\001B\n\n\010_content\"\350\004\n\004Node\022" +
+      ")\n\016outgoing_edges\030\001 \003(\0132\021.littlehorse.Ed" +
+      "ge\0228\n\020failure_handlers\030\004 \003(\0132\036.littlehor" +
+      "se.FailureHandlerDef\0221\n\nentrypoint\030\005 \001(\013" +
+      "2\033.littlehorse.EntrypointNodeH\000\022%\n\004exit\030" +
+      "\006 \001(\0132\025.littlehorse.ExitNodeH\000\022%\n\004task\030\007" +
+      " \001(\0132\025.littlehorse.TaskNodeH\000\0228\n\016externa" +
+      "l_event\030\010 \001(\0132\036.littlehorse.ExternalEven" +
+      "tNodeH\000\0224\n\014start_thread\030\t \001(\0132\034.littleho" +
+      "rse.StartThreadNodeH\000\022;\n\020wait_for_thread" +
+      "s\030\n \001(\0132\037.littlehorse.WaitForThreadsNode" +
+      "H\000\022#\n\003nop\030\013 \001(\0132\024.littlehorse.NopNodeH\000\022" +
+      "\'\n\005sleep\030\014 \001(\0132\026.littlehorse.SleepNodeH\000" +
+      "\022.\n\tuser_task\030\r \001(\0132\031.littlehorse.UserTa" +
+      "skNodeH\000\022G\n\026start_multiple_threads\030\017 \001(\013" +
+      "2%.littlehorse.StartMultipleThreadsNodeH" +
+      "\000B\006\n\004node\"\342\002\n\014UserTaskNode\022\032\n\022user_task_" +
+      "def_name\030\001 \001(\t\0228\n\nuser_group\030\002 \001(\0132\037.lit" +
+      "tlehorse.VariableAssignmentH\000\210\001\001\0225\n\007user" +
+      "_id\030\003 \001(\0132\037.littlehorse.VariableAssignme" +
+      "ntH\001\210\001\001\022-\n\007actions\030\004 \003(\0132\034.littlehorse.U" +
+      "TActionTrigger\022\"\n\025user_task_def_version\030" +
+      "\005 \001(\005H\002\210\001\001\0223\n\005notes\030\006 \001(\0132\037.littlehorse." +
+      "VariableAssignmentH\003\210\001\001B\r\n\013_user_groupB\n" +
+      "\n\010_user_idB\030\n\026_user_task_def_versionB\010\n\006" +
+      "_notes\"\233\001\n\rEdgeCondition\022+\n\ncomparator\030\001" +
+      " \001(\0162\027.littlehorse.Comparator\022-\n\004left\030\002 " +
+      "\001(\0132\037.littlehorse.VariableAssignment\022.\n\005" +
+      "right\030\003 \001(\0132\037.littlehorse.VariableAssign" +
+      "ment\"\233\001\n\004Edge\022\026\n\016sink_node_name\030\001 \001(\t\0222\n" +
+      "\tcondition\030\002 \001(\0132\032.littlehorse.EdgeCondi" +
+      "tionH\000\210\001\001\0229\n\022variable_mutations\030\003 \003(\0132\035." +
+      "littlehorse.VariableMutationB\014\n\n_conditi" +
+      "on\"\t\n\007NopNode\"\276\001\n\tSleepNode\0226\n\013raw_secon" +
+      "ds\030\001 \001(\0132\037.littlehorse.VariableAssignmen" +
+      "tH\000\0224\n\ttimestamp\030\002 \001(\0132\037.littlehorse.Var" +
+      "iableAssignmentH\000\0223\n\010iso_date\030\003 \001(\0132\037.li" +
+      "ttlehorse.VariableAssignmentH\000B\016\n\014sleep_" +
+      "length\"\207\002\n\026WfSpecVersionMigration\022\031\n\021new" +
+      "_major_version\030\001 \001(\005\022\024\n\014new_revision\030\002 \001" +
+      "(\005\022]\n\026thread_spec_migrations\030\003 \003(\0132=.lit" +
+      "tlehorse.WfSpecVersionMigration.ThreadSp" +
+      "ecMigrationsEntry\032]\n\031ThreadSpecMigration" +
+      "sEntry\022\013\n\003key\030\001 \001(\t\022/\n\005value\030\002 \001(\0132 .lit" +
+      "tlehorse.ThreadSpecMigration:\0028\001\"\325\001\n\023Thr" +
+      "eadSpecMigration\022\034\n\024new_thread_spec_name" +
+      "\030\001 \001(\t\022M\n\017node_migrations\030\002 \003(\01324.little" +
+      "horse.ThreadSpecMigration.NodeMigrations" +
+      "Entry\032Q\n\023NodeMigrationsEntry\022\013\n\003key\030\001 \001(" +
+      "\t\022)\n\005value\030\002 \001(\0132\032.littlehorse.NodeMigra" +
+      "tion:\0028\001\"&\n\rNodeMigration\022\025\n\rnew_node_na" +
+      "me\030\001 \001(\t*N\n\030WfRunVariableAccessLevel\022\016\n\n" +
+      "PUBLIC_VAR\020\000\022\017\n\013PRIVATE_VAR\020\001\022\021\n\rINHERIT" +
+      "ED_VAR\020\002BG\n\037io.littlehorse.sdk.common.pr" +
+      "otoP\001Z\007.;model\252\002\030LittleHorse.Common.Prot" +
+      "ob\006proto3"
     };
     descriptor = com.google.protobuf.Descriptors.FileDescriptor
       .internalBuildGeneratedFileFrom(descriptorData,
@@ -431,13 +439,19 @@ public static void registerAllExtensions(
     internal_static_littlehorse_WaitForThreadsNode_fieldAccessorTable = new
       com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
         internal_static_littlehorse_WaitForThreadsNode_descriptor,
-        new java.lang.String[] { "Threads", "ThreadList", "Policy", "ThreadList", });
+        new java.lang.String[] { "Threads", "ThreadList", "PerThreadFailureHandlers", "ThreadsToWaitFor", });
     internal_static_littlehorse_WaitForThreadsNode_ThreadToWaitFor_descriptor =
       internal_static_littlehorse_WaitForThreadsNode_descriptor.getNestedTypes().get(0);
     internal_static_littlehorse_WaitForThreadsNode_ThreadToWaitFor_fieldAccessorTable = new
       com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
         internal_static_littlehorse_WaitForThreadsNode_ThreadToWaitFor_descriptor,
         new java.lang.String[] { "ThreadRunNumber", });
+    internal_static_littlehorse_WaitForThreadsNode_ThreadsToWaitFor_descriptor =
+      internal_static_littlehorse_WaitForThreadsNode_descriptor.getNestedTypes().get(1);
+    internal_static_littlehorse_WaitForThreadsNode_ThreadsToWaitFor_fieldAccessorTable = new
+      com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+        internal_static_littlehorse_WaitForThreadsNode_ThreadsToWaitFor_descriptor,
+        new java.lang.String[] { "Threads", });
     internal_static_littlehorse_ExternalEventNode_descriptor =
       getDescriptor().getMessageTypes().get(12);
     internal_static_littlehorse_ExternalEventNode_fieldAccessorTable = new
diff --git a/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/WaitForThreadsNodeOutput.java b/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/WaitForThreadsNodeOutput.java
index f5f156379..55c6e7f5a 100644
--- a/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/WaitForThreadsNodeOutput.java
+++ b/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/WaitForThreadsNodeOutput.java
@@ -1,6 +1,6 @@
 package io.littlehorse.sdk.wfsdk;
 
-import io.littlehorse.sdk.common.proto.WaitForThreadsPolicy;
+import io.littlehorse.sdk.common.proto.LHErrorType;
 
 /**
  * The `WaitForThreadsNodeOutput` interface represents a specialized NodeOutput
@@ -9,25 +9,34 @@
  *
  * 
When using this interface, you can set a policy that determines how the
  * node should handle waiting for threads' parallel executions:
- * {@link WaitForThreadsPolicy#STOP_ON_FAILURE}: Stops the entire execution for parent and children threads
- * when a single node fails
  *
- * @see WaitForThreadsPolicy
  * @author Eduwer Camacaro
  */
 public interface WaitForThreadsNodeOutput extends NodeOutput {
 
     /**
-     * Sets the policy that determines how the node should handle waiting for
-     * threads' parallel executions.
-     * Usage example:
-     * WaitForThreadsNodeOutput output = thread.waitForThreads(...)
-     * output.withPolicy(WaitForThreadsPolicy.STOP_ON_FAILURE);
-     *
-     * @param policy The policy to be used by the node to determine whether to
-     *               continue with the execution or not.
-     * @return A reference to the updated WaitForThreadsNodeOutput.
-     * @see WaitForThreadsPolicy
+     * Specifies a Failure Handler to run in case any of the ThreadRun's that we are waiting for in this
+     * WaitForThreadsNode fails with a specific EXCEPTION.
+     * @param exceptionName the exception name
+     * @param handler the WorkflowThread defining the failure handler
+     * @return this WaitForThreadsNodeOutput
      */
-    WaitForThreadsNodeOutput withPolicy(WaitForThreadsPolicy policy);
+    WaitForThreadsNodeOutput handleExceptionOnChild(String exceptionName, ThreadFunc handler);
+
+    /**
+     * Specifies a Failure Handler to run in case any of the ThreadRun's that we are waiting for in this
+     * WaitForThreadsNode fails with a specific ERROR.
+     * @param error the ERROR type
+     * @param handler the WorkflowThread defining the failure handler
+     * @return this WaitForThreadsNodeOutput
+     */
+    WaitForThreadsNodeOutput handleErrorOnChild(LHErrorType error, ThreadFunc handler);
+
+    /**
+     * Specifies a Failure Handler to run in case any of the ThreadRun's that we are waiting for in this
+     * WaitForThreadsNode fails with any Failure.
+     * @param handler the WorkflowThread defining the failure handler
+     * @return this WaitForThreadsNodeOutput
+     */
+    WaitForThreadsNodeOutput handleAnyFailureOnChild(ThreadFunc handler);
 }
diff --git a/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/internal/FixedSpawnedThreads.java b/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/internal/FixedSpawnedThreads.java
index a9a0fcdfc..5027d3794 100644
--- a/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/internal/FixedSpawnedThreads.java
+++ b/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/internal/FixedSpawnedThreads.java
@@ -2,7 +2,6 @@
 
 import io.littlehorse.sdk.common.proto.VariableType;
 import io.littlehorse.sdk.common.proto.WaitForThreadsNode;
-import io.littlehorse.sdk.common.proto.WaitForThreadsPolicy;
 import io.littlehorse.sdk.wfsdk.SpawnedThread;
 import io.littlehorse.sdk.wfsdk.SpawnedThreads;
 import java.util.Collection;
@@ -29,9 +28,8 @@ public WaitForThreadsNode buildNode() {
             WaitForThreadsNode.ThreadToWaitFor threadToWaitFor = WaitForThreadsNode.ThreadToWaitFor.newBuilder()
                     .setThreadRunNumber(BuilderUtil.assignVariable(threadNumberVariable))
                     .build();
-            waitNode.addThreads(threadToWaitFor);
+            waitNode.getThreadsBuilder().addThreads(threadToWaitFor);
         }
-        waitNode.setPolicy(WaitForThreadsPolicy.STOP_ON_FAILURE);
         return waitNode.build();
     }
 }
diff --git a/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/internal/SpawnedThreadsIterator.java b/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/internal/SpawnedThreadsIterator.java
index 48f7c07d5..a1f7fbc67 100644
--- a/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/internal/SpawnedThreadsIterator.java
+++ b/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/internal/SpawnedThreadsIterator.java
@@ -2,7 +2,6 @@
 
 import io.littlehorse.sdk.common.proto.VariableType;
 import io.littlehorse.sdk.common.proto.WaitForThreadsNode;
-import io.littlehorse.sdk.common.proto.WaitForThreadsPolicy;
 import io.littlehorse.sdk.wfsdk.SpawnedThreads;
 
 public class SpawnedThreadsIterator implements SpawnedThreads {
@@ -20,7 +19,6 @@ public SpawnedThreadsIterator(final WfRunVariableImpl internalStartedThreadVar)
     public WaitForThreadsNode buildNode() {
         WaitForThreadsNode.Builder waitNode = WaitForThreadsNode.newBuilder();
         waitNode.setThreadList(BuilderUtil.assignVariable(internalStartedThreadVar));
-        waitNode.setPolicy(WaitForThreadsPolicy.STOP_ON_FAILURE);
         return waitNode.build();
     }
 }
diff --git a/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/internal/WaitForThreadsNodeOutputImpl.java b/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/internal/WaitForThreadsNodeOutputImpl.java
index 903f0ce8e..c04b80aac 100644
--- a/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/internal/WaitForThreadsNodeOutputImpl.java
+++ b/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/internal/WaitForThreadsNodeOutputImpl.java
@@ -1,9 +1,9 @@
 package io.littlehorse.sdk.wfsdk.internal;
 
-import io.littlehorse.sdk.common.proto.Node;
+import io.littlehorse.sdk.common.proto.FailureHandlerDef;
+import io.littlehorse.sdk.common.proto.LHErrorType;
 import io.littlehorse.sdk.common.proto.ThreadSpec;
-import io.littlehorse.sdk.common.proto.WaitForThreadsNode;
-import io.littlehorse.sdk.common.proto.WaitForThreadsPolicy;
+import io.littlehorse.sdk.wfsdk.ThreadFunc;
 import io.littlehorse.sdk.wfsdk.WaitForThreadsNodeOutput;
 
 class WaitForThreadsNodeOutputImpl extends NodeOutputImpl implements WaitForThreadsNodeOutput {
@@ -15,13 +15,45 @@ public WaitForThreadsNodeOutputImpl(String nodeName, WorkflowThreadImpl parent,
         this.threadSpec = threadSpec;
     }
 
-    public WaitForThreadsNodeOutputImpl withPolicy(WaitForThreadsPolicy failureStrategy) {
-        Node nodesOrThrow = threadSpec.getNodesOrThrow(nodeName);
-        WaitForThreadsNode waitForThreads = nodesOrThrow.getWaitForThreads();
-        waitForThreads = waitForThreads.toBuilder().setPolicy(failureStrategy).build();
-        nodesOrThrow =
-                nodesOrThrow.toBuilder().setWaitForThreads(waitForThreads).build();
-        threadSpec.putNodes(nodeName, nodesOrThrow);
+    public WaitForThreadsNodeOutput handleExceptionOnChild(String exceptionName, ThreadFunc handler) {
+        String threadName = "exn-handler-" + this.nodeName + "-"
+                + (exceptionName != null ? exceptionName : FailureHandlerDef.LHFailureType.FAILURE_TYPE_EXCEPTION);
+        threadName = parent.getParent().addSubThread(threadName, handler);
+        FailureHandlerDef.Builder handlerDef = FailureHandlerDef.newBuilder().setHandlerSpecName(threadName);
+        if (exceptionName != null) {
+            handlerDef.setSpecificFailure(exceptionName);
+        } else {
+            handlerDef.setAnyFailureOfType(FailureHandlerDef.LHFailureType.FAILURE_TYPE_EXCEPTION);
+        }
+
+        parent.addFailureHandlerOnWaitForThreadsNode(this, handlerDef.build());
+
+        return this;
+    }
+
+    @Override
+    public WaitForThreadsNodeOutput handleErrorOnChild(LHErrorType errorType, ThreadFunc handler) {
+        String threadName = "error-handler-" + this.nodeName + "-"
+                + (errorType != null ? errorType.name() : FailureHandlerDef.LHFailureType.FAILURE_TYPE_ERROR);
+        threadName = parent.getParent().addSubThread(threadName, handler);
+        FailureHandlerDef.Builder handlerDef = FailureHandlerDef.newBuilder().setHandlerSpecName(threadName);
+        if (errorType != null) {
+            handlerDef.setSpecificFailure(errorType.name());
+        } else {
+            handlerDef.setAnyFailureOfType(FailureHandlerDef.LHFailureType.FAILURE_TYPE_ERROR);
+        }
+
+        parent.addFailureHandlerOnWaitForThreadsNode(this, handlerDef.build());
+
+        return this;
+    }
+
+    @Override
+    public WaitForThreadsNodeOutput handleAnyFailureOnChild(ThreadFunc handler) {
+        String threadName = "failure-handler-" + this.nodeName + "-ANY_FAILURE";
+        threadName = parent.getParent().addSubThread(threadName, handler);
+        FailureHandlerDef.Builder handlerDef = FailureHandlerDef.newBuilder().setHandlerSpecName(threadName);
+        parent.addFailureHandlerOnWaitForThreadsNode(this, handlerDef.build());
         return this;
     }
 }
diff --git a/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/internal/WorkflowThreadImpl.java b/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/internal/WorkflowThreadImpl.java
index 3973dd2ac..b43f76777 100644
--- a/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/internal/WorkflowThreadImpl.java
+++ b/sdk-java/src/main/java/io/littlehorse/sdk/wfsdk/internal/WorkflowThreadImpl.java
@@ -519,6 +519,21 @@ public void addTimeoutToExtEvt(NodeOutputImpl node, int timeoutSeconds) {
         spec.putNodes(node.nodeName, n.build());
     }
 
+    public void addFailureHandlerOnWaitForThreadsNode(WaitForThreadsNodeOutputImpl node, FailureHandlerDef handler) {
+        checkIfIsActive();
+        Node.Builder n = spec.getNodesOrThrow(node.nodeName).toBuilder();
+
+        if (n.getNodeCase() != NodeCase.WAIT_FOR_THREADS) {
+            throw new IllegalStateException("orzdash this should only be a WAIT_FOR_THREADS node");
+        }
+
+        WaitForThreadsNode.Builder subBuilder = n.getWaitForThreadsBuilder();
+        subBuilder.addPerThreadFailureHandlers(handler);
+        n.setWaitForThreads(subBuilder);
+
+        spec.putNodes(node.nodeName, n.build());
+    }
+
     public void mutate(WfRunVariable lhsVar, VariableMutationType type, Object rhs) {
         checkIfIsActive();
         WfRunVariableImpl lhs = (WfRunVariableImpl) lhsVar;
diff --git a/sdk-java/src/test/java/io/littlehorse/sdk/wfsdk/internal/FixedSpawnedThreadsTest.java b/sdk-java/src/test/java/io/littlehorse/sdk/wfsdk/internal/FixedSpawnedThreadsTest.java
index 553ff619a..2e3ec2461 100644
--- a/sdk-java/src/test/java/io/littlehorse/sdk/wfsdk/internal/FixedSpawnedThreadsTest.java
+++ b/sdk-java/src/test/java/io/littlehorse/sdk/wfsdk/internal/FixedSpawnedThreadsTest.java
@@ -24,7 +24,7 @@ void setup() {
     @Test
     void shouldBuildNodeForTwoThreadsToWaitFor() {
         WaitForThreadsNode waitForThreadsNode = fixedSpawnedThreads.buildNode();
-        assertThat(waitForThreadsNode.getThreadsList()).hasSize(2);
+        assertThat(waitForThreadsNode.getThreads().getThreadsList()).hasSize(2);
     }
 
     @Test
diff --git a/sdk-java/src/test/java/io/littlehorse/sdk/wfsdk/internal/WorkflowThreadImplTest.java b/sdk-java/src/test/java/io/littlehorse/sdk/wfsdk/internal/WorkflowThreadImplTest.java
index 5a080785b..c94b26840 100644
--- a/sdk-java/src/test/java/io/littlehorse/sdk/wfsdk/internal/WorkflowThreadImplTest.java
+++ b/sdk-java/src/test/java/io/littlehorse/sdk/wfsdk/internal/WorkflowThreadImplTest.java
@@ -4,6 +4,10 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import io.littlehorse.sdk.common.proto.Comparator;
+import io.littlehorse.sdk.common.proto.FailureHandlerDef;
+import io.littlehorse.sdk.common.proto.FailureHandlerDef.FailureToCatchCase;
+import io.littlehorse.sdk.common.proto.FailureHandlerDef.LHFailureType;
+import io.littlehorse.sdk.common.proto.LHErrorType;
 import io.littlehorse.sdk.common.proto.Node;
 import io.littlehorse.sdk.common.proto.Node.NodeCase;
 import io.littlehorse.sdk.common.proto.PutWfSpecRequest;
@@ -13,7 +17,12 @@
 import io.littlehorse.sdk.common.proto.ThreadVarDef;
 import io.littlehorse.sdk.common.proto.VariableDef;
 import io.littlehorse.sdk.common.proto.VariableType;
+import io.littlehorse.sdk.common.proto.WaitForThreadsNode;
+import io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitForCase;
 import io.littlehorse.sdk.common.proto.WorkflowRetentionPolicy;
+import io.littlehorse.sdk.wfsdk.SpawnedThread;
+import io.littlehorse.sdk.wfsdk.SpawnedThreads;
+import io.littlehorse.sdk.wfsdk.WaitForThreadsNodeOutput;
 import io.littlehorse.sdk.wfsdk.WfRunVariable;
 import io.littlehorse.sdk.wfsdk.Workflow;
 import java.time.Duration;
@@ -288,4 +297,101 @@ void testTimeoutOnExternalEventNode() {
         assertThat(node.getExternalEvent().getTimeoutSeconds().getLiteralValue().getInt())
                 .isEqualTo(10);
     }
+
+    @Test
+    void testWaitForThreadsHandleAnyFailureOnChild() {
+        Workflow workflow = new WorkflowImpl("some-wf", wf -> {
+            SpawnedThread child = wf.spawnThread(childThread -> {}, "child", Map.of());
+            WaitForThreadsNodeOutput result = wf.waitForThreads(SpawnedThreads.of(child));
+            result.handleAnyFailureOnChild(handler -> {});
+        });
+        PutWfSpecRequest wfSpec = workflow.compileWorkflow();
+
+        assertThat(wfSpec.getThreadSpecsCount()).isEqualTo(3);
+        ThreadSpec entrypoint = wfSpec.getThreadSpecsOrThrow(wfSpec.getEntrypointThreadName());
+        Node node = entrypoint.getNodesOrThrow("2-threads-WAIT_FOR_THREADS");
+        WaitForThreadsNode wftn = node.getWaitForThreads();
+
+        assertThat(wftn.getPerThreadFailureHandlersCount()).isEqualTo(1);
+        assertThat(wftn.getPerThreadFailureHandlers(0).getFailureToCatchCase())
+                .isEqualTo(FailureToCatchCase.FAILURETOCATCH_NOT_SET);
+        assertThat(wftn.getPerThreadFailureHandlers(0).getHandlerSpecName())
+                .isEqualTo("failure-handler-2-threads-WAIT_FOR_THREADS-ANY_FAILURE");
+    }
+
+    @Test
+    void testWaitForThreadsHandleAnyErrorOnChild() {
+        Workflow workflow = new WorkflowImpl("some-wf", wf -> {
+            SpawnedThread child = wf.spawnThread(childThread -> {}, "child", Map.of());
+            WaitForThreadsNodeOutput result = wf.waitForThreads(SpawnedThreads.of(child));
+            result.handleErrorOnChild(LHErrorType.TIMEOUT, handler -> {});
+            result.handleErrorOnChild(null, handler -> {});
+        });
+        PutWfSpecRequest wfSpec = workflow.compileWorkflow();
+
+        assertThat(wfSpec.getThreadSpecsCount()).isEqualTo(4);
+        ThreadSpec entrypoint = wfSpec.getThreadSpecsOrThrow(wfSpec.getEntrypointThreadName());
+        Node node = entrypoint.getNodesOrThrow("2-threads-WAIT_FOR_THREADS");
+        WaitForThreadsNode wftn = node.getWaitForThreads();
+
+        assertThat(wftn.getPerThreadFailureHandlersCount()).isEqualTo(2);
+        FailureHandlerDef timeoutHandler = wftn.getPerThreadFailureHandlers(0);
+        FailureHandlerDef anyErrorHandler = wftn.getPerThreadFailureHandlers(1);
+
+        assertThat(timeoutHandler.getFailureToCatchCase()).isEqualTo(FailureToCatchCase.SPECIFIC_FAILURE);
+        assertThat(timeoutHandler.getSpecificFailure()).isEqualTo(LHErrorType.TIMEOUT.toString());
+        assertThat(timeoutHandler.getHandlerSpecName()).isEqualTo("error-handler-2-threads-WAIT_FOR_THREADS-TIMEOUT");
+
+        assertThat(anyErrorHandler.getFailureToCatchCase()).isEqualTo(FailureToCatchCase.ANY_FAILURE_OF_TYPE);
+        assertThat(anyErrorHandler.getAnyFailureOfType()).isEqualTo(LHFailureType.FAILURE_TYPE_ERROR);
+        assertThat(anyErrorHandler.getHandlerSpecName())
+                .isEqualTo("error-handler-2-threads-WAIT_FOR_THREADS-FAILURE_TYPE_ERROR");
+    }
+
+    @Test
+    void testWaitForThreadsHandleAnyExceptionOnChild() {
+        Workflow workflow = new WorkflowImpl("some-wf", wf -> {
+            SpawnedThread child = wf.spawnThread(childThread -> {}, "child", Map.of());
+            WaitForThreadsNodeOutput result = wf.waitForThreads(SpawnedThreads.of(child));
+            result.handleExceptionOnChild("my-exception", handler -> {});
+            result.handleExceptionOnChild(null, handler -> {});
+        });
+        PutWfSpecRequest wfSpec = workflow.compileWorkflow();
+
+        assertThat(wfSpec.getThreadSpecsCount()).isEqualTo(4);
+        ThreadSpec entrypoint = wfSpec.getThreadSpecsOrThrow(wfSpec.getEntrypointThreadName());
+        Node node = entrypoint.getNodesOrThrow("2-threads-WAIT_FOR_THREADS");
+        WaitForThreadsNode wftn = node.getWaitForThreads();
+
+        assertThat(wftn.getPerThreadFailureHandlersCount()).isEqualTo(2);
+        FailureHandlerDef specificHandler = wftn.getPerThreadFailureHandlers(0);
+        FailureHandlerDef anyHandler = wftn.getPerThreadFailureHandlers(1);
+
+        assertThat(specificHandler.getFailureToCatchCase()).isEqualTo(FailureToCatchCase.SPECIFIC_FAILURE);
+        assertThat(specificHandler.getSpecificFailure()).isEqualTo("my-exception");
+        assertThat(specificHandler.getHandlerSpecName())
+                .isEqualTo("exn-handler-2-threads-WAIT_FOR_THREADS-my-exception");
+
+        assertThat(anyHandler.getFailureToCatchCase()).isEqualTo(FailureToCatchCase.ANY_FAILURE_OF_TYPE);
+        assertThat(anyHandler.getAnyFailureOfType()).isEqualTo(LHFailureType.FAILURE_TYPE_EXCEPTION);
+        assertThat(anyHandler.getHandlerSpecName())
+                .isEqualTo("exn-handler-2-threads-WAIT_FOR_THREADS-FAILURE_TYPE_EXCEPTION");
+    }
+
+    @Test
+    void testWaitForParallelSpawnThreads() {
+        Workflow workflow = new WorkflowImpl("some-wf", wf -> {
+            WfRunVariable toSpawn = wf.addVariable("to-spawn", VariableType.JSON_ARR);
+            wf.waitForThreads(wf.spawnThreadForEach(toSpawn, "child", child -> {}));
+        });
+        PutWfSpecRequest wfSpec = workflow.compileWorkflow();
+
+        assertThat(wfSpec.getThreadSpecsCount()).isEqualTo(2);
+        ThreadSpec entrypoint = wfSpec.getThreadSpecsOrThrow(wfSpec.getEntrypointThreadName());
+        Node node = entrypoint.getNodesOrThrow("2-threads-WAIT_FOR_THREADS");
+        WaitForThreadsNode wftn = node.getWaitForThreads();
+
+        assertThat(wftn.getThreadsToWaitForCase()).isEqualTo(ThreadsToWaitForCase.THREAD_LIST);
+        assertThat(wftn.getThreadList().getVariableName()).isEqualTo("1-child-START_MULTIPLE_THREADS");
+    }
 }
diff --git a/sdk-python/littlehorse/model/common_enums_pb2.py b/sdk-python/littlehorse/model/common_enums_pb2.py
index cac5d70f3..ce8dfa299 100644
--- a/sdk-python/littlehorse/model/common_enums_pb2.py
+++ b/sdk-python/littlehorse/model/common_enums_pb2.py
@@ -13,7 +13,7 @@
 
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12\x63ommon_enums.proto\x12\x0blittlehorse*g\n\x08LHStatus\x12\x0c\n\x08STARTING\x10\x00\x12\x0b\n\x07RUNNING\x10\x01\x12\r\n\tCOMPLETED\x10\x02\x12\x0b\n\x07HALTING\x10\x03\x12\n\n\x06HALTED\x10\x04\x12\t\n\x05\x45RROR\x10\x05\x12\r\n\tEXCEPTION\x10\x06*;\n\x0eMetadataStatus\x12\n\n\x06\x41\x43TIVE\x10\x00\x12\x0c\n\x08\x41RCHIVED\x10\x01\x12\x0f\n\x0bTERMINATING\x10\x02*\xbc\x01\n\nTaskStatus\x12\x12\n\x0eTASK_SCHEDULED\x10\x00\x12\x10\n\x0cTASK_RUNNING\x10\x01\x12\x10\n\x0cTASK_SUCCESS\x10\x02\x12\x0f\n\x0bTASK_FAILED\x10\x03\x12\x10\n\x0cTASK_TIMEOUT\x10\x04\x12!\n\x1dTASK_OUTPUT_SERIALIZING_ERROR\x10\x05\x12\x1c\n\x18TASK_INPUT_VAR_SUB_ERROR\x10\x06\x12\x12\n\x0eTASK_EXCEPTION\x10\x08*=\n\x13MetricsWindowLength\x12\r\n\tMINUTES_5\x10\x00\x12\x0b\n\x07HOURS_2\x10\x01\x12\n\n\x06\x44\x41YS_1\x10\x02*]\n\x0cVariableType\x12\x0c\n\x08JSON_OBJ\x10\x00\x12\x0c\n\x08JSON_ARR\x10\x01\x12\n\n\x06\x44OUBLE\x10\x02\x12\x08\n\x04\x42OOL\x10\x03\x12\x07\n\x03STR\x10\x04\x12\x07\n\x03INT\x10\x05\x12\t\n\x05\x42YTES\x10\x06*\xb6\x01\n\x0bLHErrorType\x12\x11\n\rCHILD_FAILURE\x10\x00\x12\x11\n\rVAR_SUB_ERROR\x10\x01\x12\x16\n\x12VAR_MUTATION_ERROR\x10\x02\x12\x17\n\x13USER_TASK_CANCELLED\x10\x03\x12\x0b\n\x07TIMEOUT\x10\x04\x12\x10\n\x0cTASK_FAILURE\x10\x05\x12\r\n\tVAR_ERROR\x10\x06\x12\x0e\n\nTASK_ERROR\x10\x07\x12\x12\n\x0eINTERNAL_ERROR\x10\x08*+\n\x14WaitForThreadsPolicy\x12\x13\n\x0fSTOP_ON_FAILURE\x10\x00\x42G\n\x1fio.littlehorse.sdk.common.protoP\x01Z\x07.;model\xaa\x02\x18LittleHorse.Common.Protob\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12\x63ommon_enums.proto\x12\x0blittlehorse*g\n\x08LHStatus\x12\x0c\n\x08STARTING\x10\x00\x12\x0b\n\x07RUNNING\x10\x01\x12\r\n\tCOMPLETED\x10\x02\x12\x0b\n\x07HALTING\x10\x03\x12\n\n\x06HALTED\x10\x04\x12\t\n\x05\x45RROR\x10\x05\x12\r\n\tEXCEPTION\x10\x06*;\n\x0eMetadataStatus\x12\n\n\x06\x41\x43TIVE\x10\x00\x12\x0c\n\x08\x41RCHIVED\x10\x01\x12\x0f\n\x0bTERMINATING\x10\x02*\xbc\x01\n\nTaskStatus\x12\x12\n\x0eTASK_SCHEDULED\x10\x00\x12\x10\n\x0cTASK_RUNNING\x10\x01\x12\x10\n\x0cTASK_SUCCESS\x10\x02\x12\x0f\n\x0bTASK_FAILED\x10\x03\x12\x10\n\x0cTASK_TIMEOUT\x10\x04\x12!\n\x1dTASK_OUTPUT_SERIALIZING_ERROR\x10\x05\x12\x1c\n\x18TASK_INPUT_VAR_SUB_ERROR\x10\x06\x12\x12\n\x0eTASK_EXCEPTION\x10\x08*=\n\x13MetricsWindowLength\x12\r\n\tMINUTES_5\x10\x00\x12\x0b\n\x07HOURS_2\x10\x01\x12\n\n\x06\x44\x41YS_1\x10\x02*]\n\x0cVariableType\x12\x0c\n\x08JSON_OBJ\x10\x00\x12\x0c\n\x08JSON_ARR\x10\x01\x12\n\n\x06\x44OUBLE\x10\x02\x12\x08\n\x04\x42OOL\x10\x03\x12\x07\n\x03STR\x10\x04\x12\x07\n\x03INT\x10\x05\x12\t\n\x05\x42YTES\x10\x06*\xb6\x01\n\x0bLHErrorType\x12\x11\n\rCHILD_FAILURE\x10\x00\x12\x11\n\rVAR_SUB_ERROR\x10\x01\x12\x16\n\x12VAR_MUTATION_ERROR\x10\x02\x12\x17\n\x13USER_TASK_CANCELLED\x10\x03\x12\x0b\n\x07TIMEOUT\x10\x04\x12\x10\n\x0cTASK_FAILURE\x10\x05\x12\r\n\tVAR_ERROR\x10\x06\x12\x0e\n\nTASK_ERROR\x10\x07\x12\x12\n\x0eINTERNAL_ERROR\x10\x08\x42G\n\x1fio.littlehorse.sdk.common.protoP\x01Z\x07.;model\xaa\x02\x18LittleHorse.Common.Protob\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -34,6 +34,4 @@
   _globals['_VARIABLETYPE']._serialized_end=548
   _globals['_LHERRORTYPE']._serialized_start=551
   _globals['_LHERRORTYPE']._serialized_end=733
-  _globals['_WAITFORTHREADSPOLICY']._serialized_start=735
-  _globals['_WAITFORTHREADSPOLICY']._serialized_end=778
 # @@protoc_insertion_point(module_scope)
diff --git a/sdk-python/littlehorse/model/common_enums_pb2.pyi b/sdk-python/littlehorse/model/common_enums_pb2.pyi
index 296a3e843..94f7d0d01 100644
--- a/sdk-python/littlehorse/model/common_enums_pb2.pyi
+++ b/sdk-python/littlehorse/model/common_enums_pb2.pyi
@@ -58,10 +58,6 @@ class LHErrorType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     VAR_ERROR: _ClassVar[LHErrorType]
     TASK_ERROR: _ClassVar[LHErrorType]
     INTERNAL_ERROR: _ClassVar[LHErrorType]
-
-class WaitForThreadsPolicy(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
-    __slots__ = []
-    STOP_ON_FAILURE: _ClassVar[WaitForThreadsPolicy]
 STARTING: LHStatus
 RUNNING: LHStatus
 COMPLETED: LHStatus
@@ -99,4 +95,3 @@ TASK_FAILURE: LHErrorType
 VAR_ERROR: LHErrorType
 TASK_ERROR: LHErrorType
 INTERNAL_ERROR: LHErrorType
-STOP_ON_FAILURE: WaitForThreadsPolicy
diff --git a/sdk-python/littlehorse/model/node_run_pb2.py b/sdk-python/littlehorse/model/node_run_pb2.py
index 246844c41..37e93cdfa 100644
--- a/sdk-python/littlehorse/model/node_run_pb2.py
+++ b/sdk-python/littlehorse/model/node_run_pb2.py
@@ -17,7 +17,7 @@
 import littlehorse.model.object_id_pb2 as object__id__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0enode_run.proto\x12\x0blittlehorse\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x12\x63ommon_enums.proto\x1a\x0evariable.proto\x1a\x0fobject_id.proto\"\xed\x06\n\x07NodeRun\x12\"\n\x02id\x18\x01 \x01(\x0b\x32\x16.littlehorse.NodeRunId\x12)\n\nwf_spec_id\x18\x04 \x01(\x0b\x32\x15.littlehorse.WfSpecId\x12\x1b\n\x13\x66\x61ilure_handler_ids\x18\x05 \x03(\x05\x12%\n\x06status\x18\x06 \x01(\x0e\x32\x15.littlehorse.LHStatus\x12\x30\n\x0c\x61rrival_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x31\n\x08\x65nd_time\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x01\x88\x01\x01\x12\x18\n\x10thread_spec_name\x18\t \x01(\t\x12\x11\n\tnode_name\x18\n \x01(\t\x12\x1a\n\rerror_message\x18\x0b \x01(\tH\x02\x88\x01\x01\x12&\n\x08\x66\x61ilures\x18\x0c \x03(\x0b\x32\x14.littlehorse.Failure\x12(\n\x04task\x18\r \x01(\x0b\x32\x18.littlehorse.TaskNodeRunH\x00\x12\x37\n\x0e\x65xternal_event\x18\x0e \x01(\x0b\x32\x1d.littlehorse.ExternalEventRunH\x00\x12\x30\n\nentrypoint\x18\x0f \x01(\x0b\x32\x1a.littlehorse.EntrypointRunH\x00\x12$\n\x04\x65xit\x18\x10 \x01(\x0b\x32\x14.littlehorse.ExitRunH\x00\x12\x33\n\x0cstart_thread\x18\x11 \x01(\x0b\x32\x1b.littlehorse.StartThreadRunH\x00\x12\x36\n\x0cwait_threads\x18\x12 \x01(\x0b\x32\x1e.littlehorse.WaitForThreadsRunH\x00\x12*\n\x05sleep\x18\x13 \x01(\x0b\x32\x19.littlehorse.SleepNodeRunH\x00\x12\x31\n\tuser_task\x18\x14 \x01(\x0b\x32\x1c.littlehorse.UserTaskNodeRunH\x00\x12\x46\n\x16start_multiple_threads\x18\x15 \x01(\x0b\x32$.littlehorse.StartMultipleThreadsRunH\x00\x42\x0b\n\tnode_typeB\x0b\n\t_end_timeB\x10\n\x0e_error_message\"O\n\x0bTaskNodeRun\x12\x30\n\x0btask_run_id\x18\x01 \x01(\x0b\x32\x16.littlehorse.TaskRunIdH\x00\x88\x01\x01\x42\x0e\n\x0c_task_run_id\"a\n\x0fUserTaskNodeRun\x12\x39\n\x10user_task_run_id\x18\x01 \x01(\x0b\x32\x1a.littlehorse.UserTaskRunIdH\x00\x88\x01\x01\x42\x13\n\x11_user_task_run_id\"\x0f\n\rEntrypointRun\"\t\n\x07\x45xitRun\"\\\n\x0eStartThreadRun\x12\x1c\n\x0f\x63hild_thread_id\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x18\n\x10thread_spec_name\x18\x02 \x01(\tB\x12\n\x10_child_thread_id\"3\n\x17StartMultipleThreadsRun\x12\x18\n\x10thread_spec_name\x18\x01 \x01(\t\"\xc7\x02\n\x11WaitForThreadsRun\x12=\n\x07threads\x18\x01 \x03(\x0b\x32,.littlehorse.WaitForThreadsRun.WaitForThread\x12\x31\n\x06policy\x18\x02 \x01(\x0e\x32!.littlehorse.WaitForThreadsPolicy\x1a\xbf\x01\n\rWaitForThread\x12\x38\n\x0fthread_end_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00\x88\x01\x01\x12,\n\rthread_status\x18\x02 \x01(\x0e\x32\x15.littlehorse.LHStatus\x12\x19\n\x11thread_run_number\x18\x03 \x01(\x05\x12\x17\n\x0f\x61lready_handled\x18\x05 \x01(\x08\x42\x12\n\x10_thread_end_time\"\xea\x01\n\x10\x45xternalEventRun\x12>\n\x15\x65xternal_event_def_id\x18\x01 \x01(\x0b\x32\x1f.littlehorse.ExternalEventDefId\x12\x33\n\nevent_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00\x88\x01\x01\x12<\n\x11\x65xternal_event_id\x18\x03 \x01(\x0b\x32\x1c.littlehorse.ExternalEventIdH\x01\x88\x01\x01\x42\r\n\x0b_event_timeB\x14\n\x12_external_event_id\"C\n\x0cSleepNodeRun\x12\x33\n\x0fmaturation_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\x8c\x01\n\x07\x46\x61ilure\x12\x14\n\x0c\x66\x61ilure_name\x18\x01 \x01(\t\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x30\n\x07\x63ontent\x18\x03 \x01(\x0b\x32\x1a.littlehorse.VariableValueH\x00\x88\x01\x01\x12\x1c\n\x14was_properly_handled\x18\x04 \x01(\x08\x42\n\n\x08_contentBG\n\x1fio.littlehorse.sdk.common.protoP\x01Z\x07.;model\xaa\x02\x18LittleHorse.Common.Protob\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0enode_run.proto\x12\x0blittlehorse\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x12\x63ommon_enums.proto\x1a\x0evariable.proto\x1a\x0fobject_id.proto\"\xed\x06\n\x07NodeRun\x12\"\n\x02id\x18\x01 \x01(\x0b\x32\x16.littlehorse.NodeRunId\x12)\n\nwf_spec_id\x18\x04 \x01(\x0b\x32\x15.littlehorse.WfSpecId\x12\x1b\n\x13\x66\x61ilure_handler_ids\x18\x05 \x03(\x05\x12%\n\x06status\x18\x06 \x01(\x0e\x32\x15.littlehorse.LHStatus\x12\x30\n\x0c\x61rrival_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x31\n\x08\x65nd_time\x18\x08 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x01\x88\x01\x01\x12\x18\n\x10thread_spec_name\x18\t \x01(\t\x12\x11\n\tnode_name\x18\n \x01(\t\x12\x1a\n\rerror_message\x18\x0b \x01(\tH\x02\x88\x01\x01\x12&\n\x08\x66\x61ilures\x18\x0c \x03(\x0b\x32\x14.littlehorse.Failure\x12(\n\x04task\x18\r \x01(\x0b\x32\x18.littlehorse.TaskNodeRunH\x00\x12\x37\n\x0e\x65xternal_event\x18\x0e \x01(\x0b\x32\x1d.littlehorse.ExternalEventRunH\x00\x12\x30\n\nentrypoint\x18\x0f \x01(\x0b\x32\x1a.littlehorse.EntrypointRunH\x00\x12$\n\x04\x65xit\x18\x10 \x01(\x0b\x32\x14.littlehorse.ExitRunH\x00\x12\x33\n\x0cstart_thread\x18\x11 \x01(\x0b\x32\x1b.littlehorse.StartThreadRunH\x00\x12\x36\n\x0cwait_threads\x18\x12 \x01(\x0b\x32\x1e.littlehorse.WaitForThreadsRunH\x00\x12*\n\x05sleep\x18\x13 \x01(\x0b\x32\x19.littlehorse.SleepNodeRunH\x00\x12\x31\n\tuser_task\x18\x14 \x01(\x0b\x32\x1c.littlehorse.UserTaskNodeRunH\x00\x12\x46\n\x16start_multiple_threads\x18\x15 \x01(\x0b\x32$.littlehorse.StartMultipleThreadsRunH\x00\x42\x0b\n\tnode_typeB\x0b\n\t_end_timeB\x10\n\x0e_error_message\"O\n\x0bTaskNodeRun\x12\x30\n\x0btask_run_id\x18\x01 \x01(\x0b\x32\x16.littlehorse.TaskRunIdH\x00\x88\x01\x01\x42\x0e\n\x0c_task_run_id\"a\n\x0fUserTaskNodeRun\x12\x39\n\x10user_task_run_id\x18\x01 \x01(\x0b\x32\x1a.littlehorse.UserTaskRunIdH\x00\x88\x01\x01\x42\x13\n\x11_user_task_run_id\"\x0f\n\rEntrypointRun\"\t\n\x07\x45xitRun\"\\\n\x0eStartThreadRun\x12\x1c\n\x0f\x63hild_thread_id\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x18\n\x10thread_spec_name\x18\x02 \x01(\tB\x12\n\x10_child_thread_id\"3\n\x17StartMultipleThreadsRun\x12\x18\n\x10thread_spec_name\x18\x01 \x01(\t\"\xa4\x04\n\x11WaitForThreadsRun\x12=\n\x07threads\x18\x01 \x03(\x0b\x32,.littlehorse.WaitForThreadsRun.WaitForThread\x1a\xc0\x02\n\rWaitForThread\x12\x38\n\x0fthread_end_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00\x88\x01\x01\x12,\n\rthread_status\x18\x02 \x01(\x0e\x32\x15.littlehorse.LHStatus\x12\x19\n\x11thread_run_number\x18\x03 \x01(\x05\x12J\n\x0ewaiting_status\x18\x04 \x01(\x0e\x32\x32.littlehorse.WaitForThreadsRun.WaitingThreadStatus\x12*\n\x1d\x66\x61ilure_handler_thread_run_id\x18\x05 \x01(\x05H\x01\x88\x01\x01\x42\x12\n\x10_thread_end_timeB \n\x1e_failure_handler_thread_run_id\"\x8c\x01\n\x13WaitingThreadStatus\x12\x16\n\x12THREAD_IN_PROGRESS\x10\x00\x12\x1b\n\x17THREAD_HANDLING_FAILURE\x10\x01\x12\'\n#THREAD_COMPLETED_OR_FAILURE_HANDLED\x10\x02\x12\x17\n\x13THREAD_UNSUCCESSFUL\x10\x03\"\xea\x01\n\x10\x45xternalEventRun\x12>\n\x15\x65xternal_event_def_id\x18\x01 \x01(\x0b\x32\x1f.littlehorse.ExternalEventDefId\x12\x33\n\nevent_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00\x88\x01\x01\x12<\n\x11\x65xternal_event_id\x18\x03 \x01(\x0b\x32\x1c.littlehorse.ExternalEventIdH\x01\x88\x01\x01\x42\r\n\x0b_event_timeB\x14\n\x12_external_event_id\"C\n\x0cSleepNodeRun\x12\x33\n\x0fmaturation_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\x8c\x01\n\x07\x46\x61ilure\x12\x14\n\x0c\x66\x61ilure_name\x18\x01 \x01(\t\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x30\n\x07\x63ontent\x18\x03 \x01(\x0b\x32\x1a.littlehorse.VariableValueH\x00\x88\x01\x01\x12\x1c\n\x14was_properly_handled\x18\x04 \x01(\x08\x42\n\n\x08_contentBG\n\x1fio.littlehorse.sdk.common.protoP\x01Z\x07.;model\xaa\x02\x18LittleHorse.Common.Protob\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -41,13 +41,15 @@
   _globals['_STARTMULTIPLETHREADSRUN']._serialized_start=1299
   _globals['_STARTMULTIPLETHREADSRUN']._serialized_end=1350
   _globals['_WAITFORTHREADSRUN']._serialized_start=1353
-  _globals['_WAITFORTHREADSRUN']._serialized_end=1680
-  _globals['_WAITFORTHREADSRUN_WAITFORTHREAD']._serialized_start=1489
-  _globals['_WAITFORTHREADSRUN_WAITFORTHREAD']._serialized_end=1680
-  _globals['_EXTERNALEVENTRUN']._serialized_start=1683
-  _globals['_EXTERNALEVENTRUN']._serialized_end=1917
-  _globals['_SLEEPNODERUN']._serialized_start=1919
-  _globals['_SLEEPNODERUN']._serialized_end=1986
-  _globals['_FAILURE']._serialized_start=1989
-  _globals['_FAILURE']._serialized_end=2129
+  _globals['_WAITFORTHREADSRUN']._serialized_end=1901
+  _globals['_WAITFORTHREADSRUN_WAITFORTHREAD']._serialized_start=1438
+  _globals['_WAITFORTHREADSRUN_WAITFORTHREAD']._serialized_end=1758
+  _globals['_WAITFORTHREADSRUN_WAITINGTHREADSTATUS']._serialized_start=1761
+  _globals['_WAITFORTHREADSRUN_WAITINGTHREADSTATUS']._serialized_end=1901
+  _globals['_EXTERNALEVENTRUN']._serialized_start=1904
+  _globals['_EXTERNALEVENTRUN']._serialized_end=2138
+  _globals['_SLEEPNODERUN']._serialized_start=2140
+  _globals['_SLEEPNODERUN']._serialized_end=2207
+  _globals['_FAILURE']._serialized_start=2210
+  _globals['_FAILURE']._serialized_end=2350
 # @@protoc_insertion_point(module_scope)
diff --git a/sdk-python/littlehorse/model/node_run_pb2.pyi b/sdk-python/littlehorse/model/node_run_pb2.pyi
index 160e44e1f..fe9b80896 100644
--- a/sdk-python/littlehorse/model/node_run_pb2.pyi
+++ b/sdk-python/littlehorse/model/node_run_pb2.pyi
@@ -3,6 +3,7 @@ import littlehorse.model.common_enums_pb2 as _common_enums_pb2
 import littlehorse.model.variable_pb2 as _variable_pb2
 import littlehorse.model.object_id_pb2 as _object_id_pb2
 from google.protobuf.internal import containers as _containers
+from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import message as _message
 from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union
@@ -86,23 +87,33 @@ class StartMultipleThreadsRun(_message.Message):
     def __init__(self, thread_spec_name: _Optional[str] = ...) -> None: ...
 
 class WaitForThreadsRun(_message.Message):
-    __slots__ = ["threads", "policy"]
+    __slots__ = ["threads"]
+    class WaitingThreadStatus(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+        __slots__ = []
+        THREAD_IN_PROGRESS: _ClassVar[WaitForThreadsRun.WaitingThreadStatus]
+        THREAD_HANDLING_FAILURE: _ClassVar[WaitForThreadsRun.WaitingThreadStatus]
+        THREAD_COMPLETED_OR_FAILURE_HANDLED: _ClassVar[WaitForThreadsRun.WaitingThreadStatus]
+        THREAD_UNSUCCESSFUL: _ClassVar[WaitForThreadsRun.WaitingThreadStatus]
+    THREAD_IN_PROGRESS: WaitForThreadsRun.WaitingThreadStatus
+    THREAD_HANDLING_FAILURE: WaitForThreadsRun.WaitingThreadStatus
+    THREAD_COMPLETED_OR_FAILURE_HANDLED: WaitForThreadsRun.WaitingThreadStatus
+    THREAD_UNSUCCESSFUL: WaitForThreadsRun.WaitingThreadStatus
     class WaitForThread(_message.Message):
-        __slots__ = ["thread_end_time", "thread_status", "thread_run_number", "already_handled"]
+        __slots__ = ["thread_end_time", "thread_status", "thread_run_number", "waiting_status", "failure_handler_thread_run_id"]
         THREAD_END_TIME_FIELD_NUMBER: _ClassVar[int]
         THREAD_STATUS_FIELD_NUMBER: _ClassVar[int]
         THREAD_RUN_NUMBER_FIELD_NUMBER: _ClassVar[int]
-        ALREADY_HANDLED_FIELD_NUMBER: _ClassVar[int]
+        WAITING_STATUS_FIELD_NUMBER: _ClassVar[int]
+        FAILURE_HANDLER_THREAD_RUN_ID_FIELD_NUMBER: _ClassVar[int]
         thread_end_time: _timestamp_pb2.Timestamp
         thread_status: _common_enums_pb2.LHStatus
         thread_run_number: int
-        already_handled: bool
-        def __init__(self, thread_end_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., thread_status: _Optional[_Union[_common_enums_pb2.LHStatus, str]] = ..., thread_run_number: _Optional[int] = ..., already_handled: bool = ...) -> None: ...
+        waiting_status: WaitForThreadsRun.WaitingThreadStatus
+        failure_handler_thread_run_id: int
+        def __init__(self, thread_end_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., thread_status: _Optional[_Union[_common_enums_pb2.LHStatus, str]] = ..., thread_run_number: _Optional[int] = ..., waiting_status: _Optional[_Union[WaitForThreadsRun.WaitingThreadStatus, str]] = ..., failure_handler_thread_run_id: _Optional[int] = ...) -> None: ...
     THREADS_FIELD_NUMBER: _ClassVar[int]
-    POLICY_FIELD_NUMBER: _ClassVar[int]
     threads: _containers.RepeatedCompositeFieldContainer[WaitForThreadsRun.WaitForThread]
-    policy: _common_enums_pb2.WaitForThreadsPolicy
-    def __init__(self, threads: _Optional[_Iterable[_Union[WaitForThreadsRun.WaitForThread, _Mapping]]] = ..., policy: _Optional[_Union[_common_enums_pb2.WaitForThreadsPolicy, str]] = ...) -> None: ...
+    def __init__(self, threads: _Optional[_Iterable[_Union[WaitForThreadsRun.WaitForThread, _Mapping]]] = ...) -> None: ...
 
 class ExternalEventRun(_message.Message):
     __slots__ = ["external_event_def_id", "event_time", "external_event_id"]
diff --git a/sdk-python/littlehorse/model/wf_spec_pb2.py b/sdk-python/littlehorse/model/wf_spec_pb2.py
index ee86772dd..f903ebcda 100644
--- a/sdk-python/littlehorse/model/wf_spec_pb2.py
+++ b/sdk-python/littlehorse/model/wf_spec_pb2.py
@@ -17,7 +17,7 @@
 import littlehorse.model.object_id_pb2 as object__id__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rwf_spec.proto\x12\x0blittlehorse\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x12\x63ommon_enums.proto\x1a\x13\x63ommon_wfspec.proto\x1a\x0fobject_id.proto\"\xb4\x05\n\x06WfSpec\x12!\n\x02id\x18\x01 \x01(\x0b\x32\x15.littlehorse.WfSpecId\x12.\n\ncreated_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x33\n\x10\x66rozen_variables\x18\x03 \x03(\x0b\x32\x19.littlehorse.ThreadVarDef\x12+\n\x06status\x18\x04 \x01(\x0e\x32\x1b.littlehorse.MetadataStatus\x12:\n\x0cthread_specs\x18\x05 \x03(\x0b\x32$.littlehorse.WfSpec.ThreadSpecsEntry\x12\x1e\n\x16\x65ntrypoint_thread_name\x18\x06 \x01(\t\x12\x43\n\x10retention_policy\x18\x07 \x01(\x0b\x32$.littlehorse.WorkflowRetentionPolicyH\x00\x88\x01\x01\x12;\n\tmigration\x18\x08 \x01(\x0b\x32#.littlehorse.WfSpecVersionMigrationH\x01\x88\x01\x01\x12\x46\n\x0eparent_wf_spec\x18\t \x01(\x0b\x32).littlehorse.WfSpec.ParentWfSpecReferenceH\x02\x88\x01\x01\x1aK\n\x10ThreadSpecsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.littlehorse.ThreadSpec:\x02\x38\x01\x1aL\n\x15ParentWfSpecReference\x12\x14\n\x0cwf_spec_name\x18\x01 \x01(\t\x12\x1d\n\x15wf_spec_major_version\x18\x02 \x01(\x05\x42\x13\n\x11_retention_policyB\x0c\n\n_migrationB\x11\n\x0f_parent_wf_spec\"Q\n\x17WorkflowRetentionPolicy\x12&\n\x1cseconds_after_wf_termination\x18\x01 \x01(\x03H\x00\x42\x0e\n\x0cwf_gc_policy\"N\n\tJsonIndex\x12\x12\n\nfield_path\x18\x01 \x01(\t\x12-\n\nfield_type\x18\x02 \x01(\x0e\x32\x19.littlehorse.VariableType\"B\n\x15SearchableVariableDef\x12)\n\x07var_def\x18\x01 \x01(\x0b\x32\x18.littlehorse.VariableDef\"\xca\x01\n\x0cThreadVarDef\x12)\n\x07var_def\x18\x01 \x01(\x0b\x32\x18.littlehorse.VariableDef\x12\x10\n\x08required\x18\x02 \x01(\x08\x12\x12\n\nsearchable\x18\x03 \x01(\x08\x12,\n\x0cjson_indexes\x18\x04 \x03(\x0b\x32\x16.littlehorse.JsonIndex\x12;\n\x0c\x61\x63\x63\x65ss_level\x18\x05 \x01(\x0e\x32%.littlehorse.WfRunVariableAccessLevel\"\xbd\x02\n\nThreadSpec\x12\x31\n\x05nodes\x18\x01 \x03(\x0b\x32\".littlehorse.ThreadSpec.NodesEntry\x12\x30\n\rvariable_defs\x18\x02 \x03(\x0b\x32\x19.littlehorse.ThreadVarDef\x12\x31\n\x0einterrupt_defs\x18\x03 \x03(\x0b\x32\x19.littlehorse.InterruptDef\x12\x41\n\x10retention_policy\x18\x04 \x01(\x0b\x32\".littlehorse.ThreadRetentionPolicyH\x00\x88\x01\x01\x1a?\n\nNodesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12 \n\x05value\x18\x02 \x01(\x0b\x32\x11.littlehorse.Node:\x02\x38\x01\x42\x13\n\x11_retention_policy\"W\n\x15ThreadRetentionPolicy\x12*\n seconds_after_thread_termination\x18\x01 \x01(\x03H\x00\x42\x12\n\x10thread_gc_policy\"i\n\x0cInterruptDef\x12>\n\x15\x65xternal_event_def_id\x18\x01 \x01(\x0b\x32\x1f.littlehorse.ExternalEventDefId\x12\x19\n\x11handler_spec_name\x18\x02 \x01(\t\"\xbe\x01\n\x0fStartThreadNode\x12\x18\n\x10thread_spec_name\x18\x01 \x01(\t\x12>\n\tvariables\x18\x02 \x03(\x0b\x32+.littlehorse.StartThreadNode.VariablesEntry\x1aQ\n\x0eVariablesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.littlehorse.VariableAssignment:\x02\x38\x01\"\x83\x02\n\x18StartMultipleThreadsNode\x12\x18\n\x10thread_spec_name\x18\x01 \x01(\t\x12G\n\tvariables\x18\x02 \x03(\x0b\x32\x34.littlehorse.StartMultipleThreadsNode.VariablesEntry\x12\x31\n\x08iterable\x18\x03 \x01(\x0b\x32\x1f.littlehorse.VariableAssignment\x1aQ\n\x0eVariablesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.littlehorse.VariableAssignment:\x02\x38\x01\"\xf0\x01\n\x11\x46\x61ilureHandlerDef\x12\x19\n\x11handler_spec_name\x18\x02 \x01(\t\x12\x1a\n\x10specific_failure\x18\x01 \x01(\tH\x00\x12K\n\x13\x61ny_failure_of_type\x18\x03 \x01(\x0e\x32,.littlehorse.FailureHandlerDef.LHFailureTypeH\x00\"C\n\rLHFailureType\x12\x16\n\x12\x46\x41ILURE_TYPE_ERROR\x10\x00\x12\x1a\n\x16\x46\x41ILURE_TYPE_EXCEPTION\x10\x01\x42\x12\n\x10\x66\x61ilure_to_catch\"\xa3\x02\n\x12WaitForThreadsNode\x12@\n\x07threads\x18\x01 \x03(\x0b\x32/.littlehorse.WaitForThreadsNode.ThreadToWaitFor\x12\x39\n\x0bthread_list\x18\x03 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentH\x00\x88\x01\x01\x12\x31\n\x06policy\x18\x02 \x01(\x0e\x32!.littlehorse.WaitForThreadsPolicy\x1aM\n\x0fThreadToWaitFor\x12:\n\x11thread_run_number\x18\x01 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentB\x0e\n\x0c_thread_list\"\x8d\x01\n\x11\x45xternalEventNode\x12>\n\x15\x65xternal_event_def_id\x18\x01 \x01(\x0b\x32\x1f.littlehorse.ExternalEventDefId\x12\x38\n\x0ftimeout_seconds\x18\x02 \x01(\x0b\x32\x1f.littlehorse.VariableAssignment\"\x10\n\x0e\x45ntrypointNode\"M\n\x08\x45xitNode\x12\x31\n\x0b\x66\x61ilure_def\x18\x01 \x01(\x0b\x32\x17.littlehorse.FailureDefH\x00\x88\x01\x01\x42\x0e\n\x0c_failure_def\"v\n\nFailureDef\x12\x14\n\x0c\x66\x61ilure_name\x18\x01 \x01(\t\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x35\n\x07\x63ontent\x18\x03 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentH\x00\x88\x01\x01\x42\n\n\x08_content\"\xe8\x04\n\x04Node\x12)\n\x0eoutgoing_edges\x18\x01 \x03(\x0b\x32\x11.littlehorse.Edge\x12\x38\n\x10\x66\x61ilure_handlers\x18\x04 \x03(\x0b\x32\x1e.littlehorse.FailureHandlerDef\x12\x31\n\nentrypoint\x18\x05 \x01(\x0b\x32\x1b.littlehorse.EntrypointNodeH\x00\x12%\n\x04\x65xit\x18\x06 \x01(\x0b\x32\x15.littlehorse.ExitNodeH\x00\x12%\n\x04task\x18\x07 \x01(\x0b\x32\x15.littlehorse.TaskNodeH\x00\x12\x38\n\x0e\x65xternal_event\x18\x08 \x01(\x0b\x32\x1e.littlehorse.ExternalEventNodeH\x00\x12\x34\n\x0cstart_thread\x18\t \x01(\x0b\x32\x1c.littlehorse.StartThreadNodeH\x00\x12;\n\x10wait_for_threads\x18\n \x01(\x0b\x32\x1f.littlehorse.WaitForThreadsNodeH\x00\x12#\n\x03nop\x18\x0b \x01(\x0b\x32\x14.littlehorse.NopNodeH\x00\x12\'\n\x05sleep\x18\x0c \x01(\x0b\x32\x16.littlehorse.SleepNodeH\x00\x12.\n\tuser_task\x18\r \x01(\x0b\x32\x19.littlehorse.UserTaskNodeH\x00\x12G\n\x16start_multiple_threads\x18\x0f \x01(\x0b\x32%.littlehorse.StartMultipleThreadsNodeH\x00\x42\x06\n\x04node\"\xe2\x02\n\x0cUserTaskNode\x12\x1a\n\x12user_task_def_name\x18\x01 \x01(\t\x12\x38\n\nuser_group\x18\x02 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentH\x00\x88\x01\x01\x12\x35\n\x07user_id\x18\x03 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentH\x01\x88\x01\x01\x12-\n\x07\x61\x63tions\x18\x04 \x03(\x0b\x32\x1c.littlehorse.UTActionTrigger\x12\"\n\x15user_task_def_version\x18\x05 \x01(\x05H\x02\x88\x01\x01\x12\x33\n\x05notes\x18\x06 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentH\x03\x88\x01\x01\x42\r\n\x0b_user_groupB\n\n\x08_user_idB\x18\n\x16_user_task_def_versionB\x08\n\x06_notes\"\x9b\x01\n\rEdgeCondition\x12+\n\ncomparator\x18\x01 \x01(\x0e\x32\x17.littlehorse.Comparator\x12-\n\x04left\x18\x02 \x01(\x0b\x32\x1f.littlehorse.VariableAssignment\x12.\n\x05right\x18\x03 \x01(\x0b\x32\x1f.littlehorse.VariableAssignment\"\x9b\x01\n\x04\x45\x64ge\x12\x16\n\x0esink_node_name\x18\x01 \x01(\t\x12\x32\n\tcondition\x18\x02 \x01(\x0b\x32\x1a.littlehorse.EdgeConditionH\x00\x88\x01\x01\x12\x39\n\x12variable_mutations\x18\x03 \x03(\x0b\x32\x1d.littlehorse.VariableMutationB\x0c\n\n_condition\"\t\n\x07NopNode\"\xbe\x01\n\tSleepNode\x12\x36\n\x0braw_seconds\x18\x01 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentH\x00\x12\x34\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentH\x00\x12\x33\n\x08iso_date\x18\x03 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentH\x00\x42\x0e\n\x0csleep_length\"\x87\x02\n\x16WfSpecVersionMigration\x12\x19\n\x11new_major_version\x18\x01 \x01(\x05\x12\x14\n\x0cnew_revision\x18\x02 \x01(\x05\x12]\n\x16thread_spec_migrations\x18\x03 \x03(\x0b\x32=.littlehorse.WfSpecVersionMigration.ThreadSpecMigrationsEntry\x1a]\n\x19ThreadSpecMigrationsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12/\n\x05value\x18\x02 \x01(\x0b\x32 .littlehorse.ThreadSpecMigration:\x02\x38\x01\"\xd5\x01\n\x13ThreadSpecMigration\x12\x1c\n\x14new_thread_spec_name\x18\x01 \x01(\t\x12M\n\x0fnode_migrations\x18\x02 \x03(\x0b\x32\x34.littlehorse.ThreadSpecMigration.NodeMigrationsEntry\x1aQ\n\x13NodeMigrationsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.littlehorse.NodeMigration:\x02\x38\x01\"&\n\rNodeMigration\x12\x15\n\rnew_node_name\x18\x01 \x01(\t*N\n\x18WfRunVariableAccessLevel\x12\x0e\n\nPUBLIC_VAR\x10\x00\x12\x0f\n\x0bPRIVATE_VAR\x10\x01\x12\x11\n\rINHERITED_VAR\x10\x02\x42G\n\x1fio.littlehorse.sdk.common.protoP\x01Z\x07.;model\xaa\x02\x18LittleHorse.Common.Protob\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rwf_spec.proto\x12\x0blittlehorse\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x12\x63ommon_enums.proto\x1a\x13\x63ommon_wfspec.proto\x1a\x0fobject_id.proto\"\xb4\x05\n\x06WfSpec\x12!\n\x02id\x18\x01 \x01(\x0b\x32\x15.littlehorse.WfSpecId\x12.\n\ncreated_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x33\n\x10\x66rozen_variables\x18\x03 \x03(\x0b\x32\x19.littlehorse.ThreadVarDef\x12+\n\x06status\x18\x04 \x01(\x0e\x32\x1b.littlehorse.MetadataStatus\x12:\n\x0cthread_specs\x18\x05 \x03(\x0b\x32$.littlehorse.WfSpec.ThreadSpecsEntry\x12\x1e\n\x16\x65ntrypoint_thread_name\x18\x06 \x01(\t\x12\x43\n\x10retention_policy\x18\x07 \x01(\x0b\x32$.littlehorse.WorkflowRetentionPolicyH\x00\x88\x01\x01\x12;\n\tmigration\x18\x08 \x01(\x0b\x32#.littlehorse.WfSpecVersionMigrationH\x01\x88\x01\x01\x12\x46\n\x0eparent_wf_spec\x18\t \x01(\x0b\x32).littlehorse.WfSpec.ParentWfSpecReferenceH\x02\x88\x01\x01\x1aK\n\x10ThreadSpecsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12&\n\x05value\x18\x02 \x01(\x0b\x32\x17.littlehorse.ThreadSpec:\x02\x38\x01\x1aL\n\x15ParentWfSpecReference\x12\x14\n\x0cwf_spec_name\x18\x01 \x01(\t\x12\x1d\n\x15wf_spec_major_version\x18\x02 \x01(\x05\x42\x13\n\x11_retention_policyB\x0c\n\n_migrationB\x11\n\x0f_parent_wf_spec\"Q\n\x17WorkflowRetentionPolicy\x12&\n\x1cseconds_after_wf_termination\x18\x01 \x01(\x03H\x00\x42\x0e\n\x0cwf_gc_policy\"N\n\tJsonIndex\x12\x12\n\nfield_path\x18\x01 \x01(\t\x12-\n\nfield_type\x18\x02 \x01(\x0e\x32\x19.littlehorse.VariableType\"B\n\x15SearchableVariableDef\x12)\n\x07var_def\x18\x01 \x01(\x0b\x32\x18.littlehorse.VariableDef\"\xca\x01\n\x0cThreadVarDef\x12)\n\x07var_def\x18\x01 \x01(\x0b\x32\x18.littlehorse.VariableDef\x12\x10\n\x08required\x18\x02 \x01(\x08\x12\x12\n\nsearchable\x18\x03 \x01(\x08\x12,\n\x0cjson_indexes\x18\x04 \x03(\x0b\x32\x16.littlehorse.JsonIndex\x12;\n\x0c\x61\x63\x63\x65ss_level\x18\x05 \x01(\x0e\x32%.littlehorse.WfRunVariableAccessLevel\"\xbd\x02\n\nThreadSpec\x12\x31\n\x05nodes\x18\x01 \x03(\x0b\x32\".littlehorse.ThreadSpec.NodesEntry\x12\x30\n\rvariable_defs\x18\x02 \x03(\x0b\x32\x19.littlehorse.ThreadVarDef\x12\x31\n\x0einterrupt_defs\x18\x03 \x03(\x0b\x32\x19.littlehorse.InterruptDef\x12\x41\n\x10retention_policy\x18\x04 \x01(\x0b\x32\".littlehorse.ThreadRetentionPolicyH\x00\x88\x01\x01\x1a?\n\nNodesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12 \n\x05value\x18\x02 \x01(\x0b\x32\x11.littlehorse.Node:\x02\x38\x01\x42\x13\n\x11_retention_policy\"W\n\x15ThreadRetentionPolicy\x12*\n seconds_after_thread_termination\x18\x01 \x01(\x03H\x00\x42\x12\n\x10thread_gc_policy\"i\n\x0cInterruptDef\x12>\n\x15\x65xternal_event_def_id\x18\x01 \x01(\x0b\x32\x1f.littlehorse.ExternalEventDefId\x12\x19\n\x11handler_spec_name\x18\x02 \x01(\t\"\xbe\x01\n\x0fStartThreadNode\x12\x18\n\x10thread_spec_name\x18\x01 \x01(\t\x12>\n\tvariables\x18\x02 \x03(\x0b\x32+.littlehorse.StartThreadNode.VariablesEntry\x1aQ\n\x0eVariablesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.littlehorse.VariableAssignment:\x02\x38\x01\"\x83\x02\n\x18StartMultipleThreadsNode\x12\x18\n\x10thread_spec_name\x18\x01 \x01(\t\x12G\n\tvariables\x18\x02 \x03(\x0b\x32\x34.littlehorse.StartMultipleThreadsNode.VariablesEntry\x12\x31\n\x08iterable\x18\x03 \x01(\x0b\x32\x1f.littlehorse.VariableAssignment\x1aQ\n\x0eVariablesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.littlehorse.VariableAssignment:\x02\x38\x01\"\xf0\x01\n\x11\x46\x61ilureHandlerDef\x12\x19\n\x11handler_spec_name\x18\x02 \x01(\t\x12\x1a\n\x10specific_failure\x18\x01 \x01(\tH\x00\x12K\n\x13\x61ny_failure_of_type\x18\x03 \x01(\x0e\x32,.littlehorse.FailureHandlerDef.LHFailureTypeH\x00\"C\n\rLHFailureType\x12\x16\n\x12\x46\x41ILURE_TYPE_ERROR\x10\x00\x12\x1a\n\x16\x46\x41ILURE_TYPE_EXCEPTION\x10\x01\x42\x12\n\x10\x66\x61ilure_to_catch\"\x92\x03\n\x12WaitForThreadsNode\x12\x43\n\x07threads\x18\x01 \x01(\x0b\x32\x30.littlehorse.WaitForThreadsNode.ThreadsToWaitForH\x00\x12\x36\n\x0bthread_list\x18\x02 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentH\x00\x12\x43\n\x1bper_thread_failure_handlers\x18\x03 \x03(\x0b\x32\x1e.littlehorse.FailureHandlerDef\x1aM\n\x0fThreadToWaitFor\x12:\n\x11thread_run_number\x18\x01 \x01(\x0b\x32\x1f.littlehorse.VariableAssignment\x1aT\n\x10ThreadsToWaitFor\x12@\n\x07threads\x18\x01 \x03(\x0b\x32/.littlehorse.WaitForThreadsNode.ThreadToWaitForB\x15\n\x13threads_to_wait_for\"\x8d\x01\n\x11\x45xternalEventNode\x12>\n\x15\x65xternal_event_def_id\x18\x01 \x01(\x0b\x32\x1f.littlehorse.ExternalEventDefId\x12\x38\n\x0ftimeout_seconds\x18\x02 \x01(\x0b\x32\x1f.littlehorse.VariableAssignment\"\x10\n\x0e\x45ntrypointNode\"M\n\x08\x45xitNode\x12\x31\n\x0b\x66\x61ilure_def\x18\x01 \x01(\x0b\x32\x17.littlehorse.FailureDefH\x00\x88\x01\x01\x42\x0e\n\x0c_failure_def\"v\n\nFailureDef\x12\x14\n\x0c\x66\x61ilure_name\x18\x01 \x01(\t\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x35\n\x07\x63ontent\x18\x03 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentH\x00\x88\x01\x01\x42\n\n\x08_content\"\xe8\x04\n\x04Node\x12)\n\x0eoutgoing_edges\x18\x01 \x03(\x0b\x32\x11.littlehorse.Edge\x12\x38\n\x10\x66\x61ilure_handlers\x18\x04 \x03(\x0b\x32\x1e.littlehorse.FailureHandlerDef\x12\x31\n\nentrypoint\x18\x05 \x01(\x0b\x32\x1b.littlehorse.EntrypointNodeH\x00\x12%\n\x04\x65xit\x18\x06 \x01(\x0b\x32\x15.littlehorse.ExitNodeH\x00\x12%\n\x04task\x18\x07 \x01(\x0b\x32\x15.littlehorse.TaskNodeH\x00\x12\x38\n\x0e\x65xternal_event\x18\x08 \x01(\x0b\x32\x1e.littlehorse.ExternalEventNodeH\x00\x12\x34\n\x0cstart_thread\x18\t \x01(\x0b\x32\x1c.littlehorse.StartThreadNodeH\x00\x12;\n\x10wait_for_threads\x18\n \x01(\x0b\x32\x1f.littlehorse.WaitForThreadsNodeH\x00\x12#\n\x03nop\x18\x0b \x01(\x0b\x32\x14.littlehorse.NopNodeH\x00\x12\'\n\x05sleep\x18\x0c \x01(\x0b\x32\x16.littlehorse.SleepNodeH\x00\x12.\n\tuser_task\x18\r \x01(\x0b\x32\x19.littlehorse.UserTaskNodeH\x00\x12G\n\x16start_multiple_threads\x18\x0f \x01(\x0b\x32%.littlehorse.StartMultipleThreadsNodeH\x00\x42\x06\n\x04node\"\xe2\x02\n\x0cUserTaskNode\x12\x1a\n\x12user_task_def_name\x18\x01 \x01(\t\x12\x38\n\nuser_group\x18\x02 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentH\x00\x88\x01\x01\x12\x35\n\x07user_id\x18\x03 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentH\x01\x88\x01\x01\x12-\n\x07\x61\x63tions\x18\x04 \x03(\x0b\x32\x1c.littlehorse.UTActionTrigger\x12\"\n\x15user_task_def_version\x18\x05 \x01(\x05H\x02\x88\x01\x01\x12\x33\n\x05notes\x18\x06 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentH\x03\x88\x01\x01\x42\r\n\x0b_user_groupB\n\n\x08_user_idB\x18\n\x16_user_task_def_versionB\x08\n\x06_notes\"\x9b\x01\n\rEdgeCondition\x12+\n\ncomparator\x18\x01 \x01(\x0e\x32\x17.littlehorse.Comparator\x12-\n\x04left\x18\x02 \x01(\x0b\x32\x1f.littlehorse.VariableAssignment\x12.\n\x05right\x18\x03 \x01(\x0b\x32\x1f.littlehorse.VariableAssignment\"\x9b\x01\n\x04\x45\x64ge\x12\x16\n\x0esink_node_name\x18\x01 \x01(\t\x12\x32\n\tcondition\x18\x02 \x01(\x0b\x32\x1a.littlehorse.EdgeConditionH\x00\x88\x01\x01\x12\x39\n\x12variable_mutations\x18\x03 \x03(\x0b\x32\x1d.littlehorse.VariableMutationB\x0c\n\n_condition\"\t\n\x07NopNode\"\xbe\x01\n\tSleepNode\x12\x36\n\x0braw_seconds\x18\x01 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentH\x00\x12\x34\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentH\x00\x12\x33\n\x08iso_date\x18\x03 \x01(\x0b\x32\x1f.littlehorse.VariableAssignmentH\x00\x42\x0e\n\x0csleep_length\"\x87\x02\n\x16WfSpecVersionMigration\x12\x19\n\x11new_major_version\x18\x01 \x01(\x05\x12\x14\n\x0cnew_revision\x18\x02 \x01(\x05\x12]\n\x16thread_spec_migrations\x18\x03 \x03(\x0b\x32=.littlehorse.WfSpecVersionMigration.ThreadSpecMigrationsEntry\x1a]\n\x19ThreadSpecMigrationsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12/\n\x05value\x18\x02 \x01(\x0b\x32 .littlehorse.ThreadSpecMigration:\x02\x38\x01\"\xd5\x01\n\x13ThreadSpecMigration\x12\x1c\n\x14new_thread_spec_name\x18\x01 \x01(\t\x12M\n\x0fnode_migrations\x18\x02 \x03(\x0b\x32\x34.littlehorse.ThreadSpecMigration.NodeMigrationsEntry\x1aQ\n\x13NodeMigrationsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.littlehorse.NodeMigration:\x02\x38\x01\"&\n\rNodeMigration\x12\x15\n\rnew_node_name\x18\x01 \x01(\t*N\n\x18WfRunVariableAccessLevel\x12\x0e\n\nPUBLIC_VAR\x10\x00\x12\x0f\n\x0bPRIVATE_VAR\x10\x01\x12\x11\n\rINHERITED_VAR\x10\x02\x42G\n\x1fio.littlehorse.sdk.common.protoP\x01Z\x07.;model\xaa\x02\x18LittleHorse.Common.Protob\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -38,8 +38,8 @@
   _WFSPECVERSIONMIGRATION_THREADSPECMIGRATIONSENTRY._serialized_options = b'8\001'
   _THREADSPECMIGRATION_NODEMIGRATIONSENTRY._options = None
   _THREADSPECMIGRATION_NODEMIGRATIONSENTRY._serialized_options = b'8\001'
-  _globals['_WFRUNVARIABLEACCESSLEVEL']._serialized_start=5139
-  _globals['_WFRUNVARIABLEACCESSLEVEL']._serialized_end=5217
+  _globals['_WFRUNVARIABLEACCESSLEVEL']._serialized_start=5250
+  _globals['_WFRUNVARIABLEACCESSLEVEL']._serialized_end=5328
   _globals['_WFSPEC']._serialized_start=122
   _globals['_WFSPEC']._serialized_end=814
   _globals['_WFSPEC_THREADSPECSENTRY']._serialized_start=607
@@ -75,37 +75,39 @@
   _globals['_FAILUREHANDLERDEF_LHFAILURETYPE']._serialized_start=2377
   _globals['_FAILUREHANDLERDEF_LHFAILURETYPE']._serialized_end=2444
   _globals['_WAITFORTHREADSNODE']._serialized_start=2467
-  _globals['_WAITFORTHREADSNODE']._serialized_end=2758
-  _globals['_WAITFORTHREADSNODE_THREADTOWAITFOR']._serialized_start=2665
-  _globals['_WAITFORTHREADSNODE_THREADTOWAITFOR']._serialized_end=2742
-  _globals['_EXTERNALEVENTNODE']._serialized_start=2761
-  _globals['_EXTERNALEVENTNODE']._serialized_end=2902
-  _globals['_ENTRYPOINTNODE']._serialized_start=2904
-  _globals['_ENTRYPOINTNODE']._serialized_end=2920
-  _globals['_EXITNODE']._serialized_start=2922
-  _globals['_EXITNODE']._serialized_end=2999
-  _globals['_FAILUREDEF']._serialized_start=3001
-  _globals['_FAILUREDEF']._serialized_end=3119
-  _globals['_NODE']._serialized_start=3122
-  _globals['_NODE']._serialized_end=3738
-  _globals['_USERTASKNODE']._serialized_start=3741
-  _globals['_USERTASKNODE']._serialized_end=4095
-  _globals['_EDGECONDITION']._serialized_start=4098
-  _globals['_EDGECONDITION']._serialized_end=4253
-  _globals['_EDGE']._serialized_start=4256
-  _globals['_EDGE']._serialized_end=4411
-  _globals['_NOPNODE']._serialized_start=4413
-  _globals['_NOPNODE']._serialized_end=4422
-  _globals['_SLEEPNODE']._serialized_start=4425
-  _globals['_SLEEPNODE']._serialized_end=4615
-  _globals['_WFSPECVERSIONMIGRATION']._serialized_start=4618
-  _globals['_WFSPECVERSIONMIGRATION']._serialized_end=4881
-  _globals['_WFSPECVERSIONMIGRATION_THREADSPECMIGRATIONSENTRY']._serialized_start=4788
-  _globals['_WFSPECVERSIONMIGRATION_THREADSPECMIGRATIONSENTRY']._serialized_end=4881
-  _globals['_THREADSPECMIGRATION']._serialized_start=4884
-  _globals['_THREADSPECMIGRATION']._serialized_end=5097
-  _globals['_THREADSPECMIGRATION_NODEMIGRATIONSENTRY']._serialized_start=5016
-  _globals['_THREADSPECMIGRATION_NODEMIGRATIONSENTRY']._serialized_end=5097
-  _globals['_NODEMIGRATION']._serialized_start=5099
-  _globals['_NODEMIGRATION']._serialized_end=5137
+  _globals['_WAITFORTHREADSNODE']._serialized_end=2869
+  _globals['_WAITFORTHREADSNODE_THREADTOWAITFOR']._serialized_start=2683
+  _globals['_WAITFORTHREADSNODE_THREADTOWAITFOR']._serialized_end=2760
+  _globals['_WAITFORTHREADSNODE_THREADSTOWAITFOR']._serialized_start=2762
+  _globals['_WAITFORTHREADSNODE_THREADSTOWAITFOR']._serialized_end=2846
+  _globals['_EXTERNALEVENTNODE']._serialized_start=2872
+  _globals['_EXTERNALEVENTNODE']._serialized_end=3013
+  _globals['_ENTRYPOINTNODE']._serialized_start=3015
+  _globals['_ENTRYPOINTNODE']._serialized_end=3031
+  _globals['_EXITNODE']._serialized_start=3033
+  _globals['_EXITNODE']._serialized_end=3110
+  _globals['_FAILUREDEF']._serialized_start=3112
+  _globals['_FAILUREDEF']._serialized_end=3230
+  _globals['_NODE']._serialized_start=3233
+  _globals['_NODE']._serialized_end=3849
+  _globals['_USERTASKNODE']._serialized_start=3852
+  _globals['_USERTASKNODE']._serialized_end=4206
+  _globals['_EDGECONDITION']._serialized_start=4209
+  _globals['_EDGECONDITION']._serialized_end=4364
+  _globals['_EDGE']._serialized_start=4367
+  _globals['_EDGE']._serialized_end=4522
+  _globals['_NOPNODE']._serialized_start=4524
+  _globals['_NOPNODE']._serialized_end=4533
+  _globals['_SLEEPNODE']._serialized_start=4536
+  _globals['_SLEEPNODE']._serialized_end=4726
+  _globals['_WFSPECVERSIONMIGRATION']._serialized_start=4729
+  _globals['_WFSPECVERSIONMIGRATION']._serialized_end=4992
+  _globals['_WFSPECVERSIONMIGRATION_THREADSPECMIGRATIONSENTRY']._serialized_start=4899
+  _globals['_WFSPECVERSIONMIGRATION_THREADSPECMIGRATIONSENTRY']._serialized_end=4992
+  _globals['_THREADSPECMIGRATION']._serialized_start=4995
+  _globals['_THREADSPECMIGRATION']._serialized_end=5208
+  _globals['_THREADSPECMIGRATION_NODEMIGRATIONSENTRY']._serialized_start=5127
+  _globals['_THREADSPECMIGRATION_NODEMIGRATIONSENTRY']._serialized_end=5208
+  _globals['_NODEMIGRATION']._serialized_start=5210
+  _globals['_NODEMIGRATION']._serialized_end=5248
 # @@protoc_insertion_point(module_scope)
diff --git a/sdk-python/littlehorse/model/wf_spec_pb2.pyi b/sdk-python/littlehorse/model/wf_spec_pb2.pyi
index 6bf17c51d..a1a46b40d 100644
--- a/sdk-python/littlehorse/model/wf_spec_pb2.pyi
+++ b/sdk-python/littlehorse/model/wf_spec_pb2.pyi
@@ -171,19 +171,24 @@ class FailureHandlerDef(_message.Message):
     def __init__(self, handler_spec_name: _Optional[str] = ..., specific_failure: _Optional[str] = ..., any_failure_of_type: _Optional[_Union[FailureHandlerDef.LHFailureType, str]] = ...) -> None: ...
 
 class WaitForThreadsNode(_message.Message):
-    __slots__ = ["threads", "thread_list", "policy"]
+    __slots__ = ["threads", "thread_list", "per_thread_failure_handlers"]
     class ThreadToWaitFor(_message.Message):
         __slots__ = ["thread_run_number"]
         THREAD_RUN_NUMBER_FIELD_NUMBER: _ClassVar[int]
         thread_run_number: _common_wfspec_pb2.VariableAssignment
         def __init__(self, thread_run_number: _Optional[_Union[_common_wfspec_pb2.VariableAssignment, _Mapping]] = ...) -> None: ...
+    class ThreadsToWaitFor(_message.Message):
+        __slots__ = ["threads"]
+        THREADS_FIELD_NUMBER: _ClassVar[int]
+        threads: _containers.RepeatedCompositeFieldContainer[WaitForThreadsNode.ThreadToWaitFor]
+        def __init__(self, threads: _Optional[_Iterable[_Union[WaitForThreadsNode.ThreadToWaitFor, _Mapping]]] = ...) -> None: ...
     THREADS_FIELD_NUMBER: _ClassVar[int]
     THREAD_LIST_FIELD_NUMBER: _ClassVar[int]
-    POLICY_FIELD_NUMBER: _ClassVar[int]
-    threads: _containers.RepeatedCompositeFieldContainer[WaitForThreadsNode.ThreadToWaitFor]
+    PER_THREAD_FAILURE_HANDLERS_FIELD_NUMBER: _ClassVar[int]
+    threads: WaitForThreadsNode.ThreadsToWaitFor
     thread_list: _common_wfspec_pb2.VariableAssignment
-    policy: _common_enums_pb2.WaitForThreadsPolicy
-    def __init__(self, threads: _Optional[_Iterable[_Union[WaitForThreadsNode.ThreadToWaitFor, _Mapping]]] = ..., thread_list: _Optional[_Union[_common_wfspec_pb2.VariableAssignment, _Mapping]] = ..., policy: _Optional[_Union[_common_enums_pb2.WaitForThreadsPolicy, str]] = ...) -> None: ...
+    per_thread_failure_handlers: _containers.RepeatedCompositeFieldContainer[FailureHandlerDef]
+    def __init__(self, threads: _Optional[_Union[WaitForThreadsNode.ThreadsToWaitFor, _Mapping]] = ..., thread_list: _Optional[_Union[_common_wfspec_pb2.VariableAssignment, _Mapping]] = ..., per_thread_failure_handlers: _Optional[_Iterable[_Union[FailureHandlerDef, _Mapping]]] = ...) -> None: ...
 
 class ExternalEventNode(_message.Message):
     __slots__ = ["external_event_def_id", "timeout_seconds"]
diff --git a/sdk-python/littlehorse/workflow.py b/sdk-python/littlehorse/workflow.py
index 730e78dc4..add673c61 100644
--- a/sdk-python/littlehorse/workflow.py
+++ b/sdk-python/littlehorse/workflow.py
@@ -10,10 +10,7 @@
 from google.protobuf.json_format import MessageToJson
 from google.protobuf.message import Message
 from littlehorse.config import LHConfig
-from littlehorse.model.common_enums_pb2 import (
-    VariableType,
-    WaitForThreadsPolicy,
-)
+from littlehorse.model.common_enums_pb2 import VariableType
 from littlehorse.model.common_wfspec_pb2 import (
     Comparator,
     TaskNode,
@@ -566,7 +563,7 @@ def build_fixed_threads(
                     )
                     threads.append(thread_to_wait_for)
             return WaitForThreadsNode(
-                threads=threads, policy=WaitForThreadsPolicy.STOP_ON_FAILURE
+                threads=WaitForThreadsNode.ThreadsToWaitFor(threads),
             )
 
         def build_iterator_threads(
@@ -574,7 +571,6 @@ def build_iterator_threads(
         ) -> WaitForThreadsNode:
             return WaitForThreadsNode(
                 thread_list=to_variable_assignment(iterable),
-                policy=WaitForThreadsPolicy.STOP_ON_FAILURE,
             )
 
         return (
diff --git a/server/src/main/java/io/littlehorse/common/exceptions/LHApiException.java b/server/src/main/java/io/littlehorse/common/exceptions/LHApiException.java
index 890a33f7d..1f6e162d2 100644
--- a/server/src/main/java/io/littlehorse/common/exceptions/LHApiException.java
+++ b/server/src/main/java/io/littlehorse/common/exceptions/LHApiException.java
@@ -22,6 +22,10 @@ public LHApiException(Status status, Throwable cause) {
     }
 
     public LHApiException getCopyWithPrefix(String prefix) {
-        return new LHApiException(getStatus().withCause(getCause()), prefix + getMessage());
+        if (!prefix.endsWith(": ")) {
+            prefix = prefix + ": ";
+        }
+        return new LHApiException(
+                getStatus().withCause(getCause()), prefix + getStatus().getDescription());
     }
 }
diff --git a/server/src/main/java/io/littlehorse/common/model/corecommand/subcommand/PutExternalEventRequestModel.java b/server/src/main/java/io/littlehorse/common/model/corecommand/subcommand/PutExternalEventRequestModel.java
index bb5171bed..d84b85302 100644
--- a/server/src/main/java/io/littlehorse/common/model/corecommand/subcommand/PutExternalEventRequestModel.java
+++ b/server/src/main/java/io/littlehorse/common/model/corecommand/subcommand/PutExternalEventRequestModel.java
@@ -95,6 +95,7 @@ public ExternalEvent process(ProcessorExecutionContext executionContext, LHServe
             WfSpecModel spec = service.getWfSpec(wfRun.getWfSpecId());
             if (spec == null) {
                 wfRun.getThreadRun(0)
+                        .getCurrentNodeRun()
                         .fail(new FailureModel("Appears wfSpec was deleted", LHConstants.INTERNAL_ERROR), new Date());
 
                 // NOTE: need to commit the dao before we throw the exception.
diff --git a/server/src/main/java/io/littlehorse/common/model/corecommand/subcommand/RunWfRequestModel.java b/server/src/main/java/io/littlehorse/common/model/corecommand/subcommand/RunWfRequestModel.java
index 13d071022..21bacf434 100644
--- a/server/src/main/java/io/littlehorse/common/model/corecommand/subcommand/RunWfRequestModel.java
+++ b/server/src/main/java/io/littlehorse/common/model/corecommand/subcommand/RunWfRequestModel.java
@@ -5,11 +5,13 @@
 import io.littlehorse.common.LHSerializable;
 import io.littlehorse.common.LHServerConfig;
 import io.littlehorse.common.exceptions.LHApiException;
+import io.littlehorse.common.exceptions.LHValidationError;
 import io.littlehorse.common.model.corecommand.CoreSubCommand;
 import io.littlehorse.common.model.getable.core.variable.VariableValueModel;
 import io.littlehorse.common.model.getable.core.wfrun.WfRunModel;
 import io.littlehorse.common.model.getable.global.wfspec.ParentWfSpecReferenceModel;
 import io.littlehorse.common.model.getable.global.wfspec.WfSpecModel;
+import io.littlehorse.common.model.getable.global.wfspec.thread.ThreadSpecModel;
 import io.littlehorse.common.model.getable.objectId.WfRunIdModel;
 import io.littlehorse.common.util.LHUtil;
 import io.littlehorse.sdk.common.proto.RunWfRequest;
@@ -133,6 +135,14 @@ public WfRun process(ProcessorExecutionContext processorContext, LHServerConfig
             }
         }
 
+        // Validate input variables before saving anything.
+        ThreadSpecModel entrypointThread = spec.getEntrypointThread();
+        try {
+            entrypointThread.validateStartVariables(variables);
+        } catch (LHValidationError exn) {
+            throw new LHApiException(Status.INVALID_ARGUMENT, exn.getMessage());
+        }
+
         WfRunModel newRun = spec.startNewRun(this, processorContext);
         newRun.advance(processorContext.currentCommand().getTime());
 
diff --git a/server/src/main/java/io/littlehorse/common/model/getable/core/noderun/NodeRunModel.java b/server/src/main/java/io/littlehorse/common/model/getable/core/noderun/NodeRunModel.java
index 1a0534526..6621bd57b 100644
--- a/server/src/main/java/io/littlehorse/common/model/getable/core/noderun/NodeRunModel.java
+++ b/server/src/main/java/io/littlehorse/common/model/getable/core/noderun/NodeRunModel.java
@@ -371,6 +371,10 @@ public boolean canBeInterrupted() {
         return getSubNodeRun().canBeInterrupted();
     }
 
+    /**
+     * Tries to advance the NodeRun, and returns true if the status of something
+     * changes.
+     */
     public boolean advanceIfPossible(Date time) {
         if (isCompletedOrRecoveredFromFailure()) {
             getThreadRun().advanceFrom(getNode());
diff --git a/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/SubNodeRun.java b/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/SubNodeRun.java
index 4342cabc1..9669b5e93 100644
--- a/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/SubNodeRun.java
+++ b/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/SubNodeRun.java
@@ -15,6 +15,10 @@ public abstract class SubNodeRun extends LHSerializable {
 
     public NodeRunModel nodeRun;
 
+    /*
+     * Tries to move forward. Returns true if the status of something in the noderun changed. That means
+     * the WfRunModel#advance() method will try to call advance again on everything.
+     */
     public abstract boolean advanceIfPossible(Date time);
 
     public abstract void arrive(Date time);
diff --git a/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/ThreadRunModel.java b/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/ThreadRunModel.java
index 58d9c8624..7577faa14 100644
--- a/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/ThreadRunModel.java
+++ b/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/ThreadRunModel.java
@@ -3,6 +3,7 @@
 import com.google.protobuf.Message;
 import io.littlehorse.common.LHConstants;
 import io.littlehorse.common.LHSerializable;
+import io.littlehorse.common.exceptions.LHValidationError;
 import io.littlehorse.common.exceptions.LHVarSubError;
 import io.littlehorse.common.model.corecommand.subcommand.ExternalEventTimeoutModel;
 import io.littlehorse.common.model.corecommand.subcommand.SleepNodeMaturedModel;
@@ -182,6 +183,7 @@ public static ThreadRunModel fromProto(Message p, ExecutionContext context) {
         return out;
     }
 
+    @Override
     public Class getProtoBaseClass() {
         return ThreadRun.class;
     }
@@ -213,6 +215,71 @@ public NodeRunModel getCurrentNodeRun() {
         return getNodeRun(currentNodePosition);
     }
 
+    public void validateVariablesAndStart(Map variables) {
+        if (currentNodePosition > 0) {
+            throw new IllegalStateException("Should only be called on creation");
+        }
+        currentNodePosition = 0;
+
+        Date now = new Date();
+        ThreadSpecModel threadSpec = getThreadSpecModel();
+        setStatus(LHStatus.RUNNING);
+        setStartTime(now);
+
+        NodeModel entrypointNode = threadSpec.getNodes().get(threadSpec.getEntrypointNodeName());
+
+        NodeRunModel entrypointRun = new NodeRunModel(processorContext);
+        entrypointRun.setThreadRun(this);
+        entrypointRun.setNodeName(entrypointNode.name);
+        entrypointRun.setStatus(LHStatus.STARTING);
+        entrypointRun.setId(new NodeRunIdModel(wfRun.getId(), this.number, 0));
+        entrypointRun.setWfSpecId(wfSpecId);
+        entrypointRun.setThreadSpecName(threadSpecName);
+        entrypointRun.setArrivalTime(now);
+        entrypointRun.setSubNodeRun(entrypointNode.getSubNode().createSubNodeRun(now));
+        putNodeRun(entrypointRun);
+
+        try {
+            threadSpec.validateStartVariables(variables);
+        } catch (LHValidationError exn) {
+            log.error("Invalid variables received", exn);
+            // TODO: determine how observability events should look like for this case.
+            entrypointRun.fail(
+                    new FailureModel(
+                            "Failed validating variables on start: " + exn.getMessage(),
+                            LHConstants.VAR_MUTATION_ERROR),
+                    now);
+            return;
+        }
+
+        for (ThreadVarDefModel threadVarDef : threadSpec.getVariableDefs()) {
+            VariableDefModel varDef = threadVarDef.getVarDef();
+            String varName = varDef.getName();
+            VariableValueModel val;
+
+            if (threadVarDef.getAccessLevel() == WfRunVariableAccessLevel.INHERITED_VAR) {
+                // We do NOT create a variable since we want to use the one from the parent.
+                continue;
+            }
+
+            if (variables.containsKey(varName)) {
+                val = variables.get(varName);
+            } else if (varDef.getDefaultValue() != null) {
+                val = varDef.getDefaultValue();
+            } else {
+                // TODO: Will need to update this when we add the required variable feature.
+                val = new VariableValueModel();
+            }
+
+            VariableModel variable = new VariableModel(varName, val, wfRun.getId(), getNumber(), wfRun.getWfSpec());
+            processorContext.getableManager().put(variable);
+        }
+
+        entrypointRun.setStatus(LHStatus.RUNNING);
+        entrypointRun.getSubNodeRun().arrive(now);
+        entrypointRun.getSubNodeRun().advanceIfPossible(now);
+    }
+
     /*
      * Note on how ExternalEvents are handled:
      * 1. First, the ExternalEvent is saved to the data store. This is handled
@@ -423,11 +490,15 @@ public boolean canBeInterrupted() {
     /*
      * Returns true if this thread is in a dynamic (running) state.
      */
-
     public boolean isRunning() {
         return (status == LHStatus.RUNNING || status == LHStatus.STARTING || status == LHStatus.HALTING);
     }
 
+    /**
+     * Tells the ThreadRun to try to move forward. Returns true if the state of the ThreadRun is changed,
+     * meaning that the ThreadRun either advanced to a new NodeRun, changed its status (eg. from HALTING to
+     * HALTED).
+     */
     public boolean advance(Date eventTime) {
         NodeRunModel currentNodeRunModel = getCurrentNodeRun();
 
@@ -438,14 +509,14 @@ public boolean advance(Date eventTime) {
             // This means we just need to wait until advance() is called again
             // after Thread Resumption
 
-            log.info("Tried to advance HALTED thread. Doing nothing.");
+            log.trace("Tried to advance HALTED thread. Doing nothing.");
             return false;
         } else if (status == LHStatus.HALTING) {
-            log.info("Tried to advance HALTING thread, checking if halted yet.");
+            log.trace("Tried to advance HALTING thread, checking if halted yet.");
 
             if (currentNodeRunModel.canBeInterrupted()) {
                 setStatus(LHStatus.HALTED);
-                log.info("Moving thread to HALTED");
+                log.trace("Moving thread to HALTED");
                 return true;
             } else {
                 return false;
@@ -466,11 +537,17 @@ public boolean advance(Date eventTime) {
         }
     }
 
+    /**
+     * Makes the ThreadRun fail with a given Failure. If the current NodeRun
+     * @param failure is the Failure that was raised.
+     * @param time when the Failure occurred.
+     */
     public void fail(FailureModel failure, Date time) {
         // First determine if the node that was failed has a relevant exception
         // handler attached.
 
         NodeModel curNode = getCurrentNode();
+
         FailureHandlerDefModel handler = null;
 
         for (FailureHandlerDefModel candidate : curNode.failureHandlers) {
@@ -481,17 +558,33 @@ public void fail(FailureModel failure, Date time) {
         }
 
         if (handler == null) {
-            dieForReal(failure, time);
+            failWithoutGrace(failure, time);
         } else {
             handleFailure(failure, handler);
         }
     }
 
+    /**
+     * In the case that a Failure is thrown and there is a FailureHandler defined for that failure,
+     * we start a "Failure Handler ThreadRun" which handles that failure.
+     * @param failure is the failure being handled
+     * @param handler is the FailureHandlerDef that defines the ThreadSpec to handle the failure.
+     */
     private void handleFailure(FailureModel failure, FailureHandlerDefModel handler) {
         PendingFailureHandlerModel pfh = new PendingFailureHandlerModel();
         pfh.failedThreadRun = this.number;
         pfh.handlerSpecName = handler.handlerSpecName;
 
+        /*
+         * It should be noted that the current implementation of Failure Handling is as follows:
+         * - We HALT the ThreadRun that threw the Failure
+         * - Once that ThreadRun is HALTED, we start the FailureHandler ThreadRun.
+         *
+         * Note that for a ThreadRun to be HALTED, we need to wait for all of its children to be
+         * HALTED as well. That is why we add the "pending failure" to the WfRun, which means that
+         * the next time we call advance(), we check to see if the failed ThreadRun is HALTED, and
+         * only the do we start the FailureHandler.
+         */
         wfRun.pendingFailures.add(pfh);
 
         ThreadHaltReasonModel haltReason = new ThreadHaltReasonModel();
@@ -504,6 +597,14 @@ private void handleFailure(FailureModel failure, FailureHandlerDefModel handler)
         getWfRun().advance(processorContext.currentCommand().getTime());
     }
 
+    /**
+     * See the note inside handleFailure(). The WfRunModel is in charge of starting the FailureHandler
+     * ThreadRun once the ThreadRun that threw the Failure has reached the HALTED state.
+     *
+     * When the WfRunModel starts the FailureHandler ThreadRun, then it must also tell the failed
+     * ThreadRunModel to "update" its state to reflect the fact that the failed ThreadRun is no longer
+     * waiting for the FailureHandler to start; rather, the FailureHandler has already started.
+     */
     public void acknowledgeXnHandlerStarted(PendingFailureHandlerModel pfh, int handlerThreadNumber) {
         boolean foundIt = false;
         for (int i = haltReasons.size() - 1; i >= 0; i--) {
@@ -528,7 +629,17 @@ public void acknowledgeXnHandlerStarted(PendingFailureHandlerModel pfh, int hand
         haltReasons.add(thr);
     }
 
-    public void dieForReal(FailureModel failure, Date time) {
+    /**
+     * Bypasses failure handlers and causes the ThreadRun to fail without possibility for handling
+     * the Failure. This is used for exmaple if the FailureHandler ThreadRun throws a failure.
+     *
+     * When a ThreadRun fails, the current behavior of the LH Server is that all child ThreadRun's
+     * are moved to the HALTED state.
+     *
+     * If the failing ThreadRun is an Interrupt Handler ThreadRun, then the parent ThreadRun is marked
+     * as failed as well.
+     */
+    public void failWithoutGrace(FailureModel failure, Date time) {
         this.errorMessage = failure.message;
         this.status = failure.getStatus();
         this.endTime = time;
@@ -546,23 +657,27 @@ public void dieForReal(FailureModel failure, Date time) {
         }
 
         if (interruptTriggerId != null) {
-            // then we're an interrupt thread and need to fail the parent.
-
-            getParent() // guaranteed not to be null in this case
+            // then we're an interrupt thread and need to fail the parent. Parent is guaranteed to
+            // to be not-null in this case
+            getParent()
                     .failWithoutGrace(
-                            new FailureModel("Interrupt thread with id " + number + " failed!", failure.failureName),
+                            new FailureModel(
+                                    "Interrupt thread with id " + number + " failed!",
+                                    failure.getFailureName(),
+                                    failure.getContent()), // propagate failure content
                             time);
         } else if (failureBeingHandled != null) {
+            // Then it's a FailureHandler thread, so we want the parent ThreadRun to fail without
+            // grace.
             getParent().failWithoutGrace(failure, time);
         }
 
         wfRun.handleThreadStatus(number, new Date(), status);
     }
 
-    public void failWithoutGrace(FailureModel failure, Date time) {
-        dieForReal(failure, time);
-    }
-
+    /*
+     * Marks the ThreadRun as completed and notifies the WfRunModel as so.
+     */
     public void complete(Date time) {
         this.errorMessage = null;
         setStatus(LHStatus.COMPLETED);
@@ -571,6 +686,13 @@ public void complete(Date time) {
         wfRun.handleThreadStatus(number, new Date(), status);
     }
 
+    /*
+     * Callback used by the NodeRunModel when the NodeRun is completed, to notify this ThreadRunModel
+     * that it's time to advance the ThreadRun.
+     *
+     * This callback makes the ThreadRun advance to the next Node in the WfSpec (thus creating a new
+     * NodeRun) and it tells the WfRun to try to advance everything.
+     */
     public void completeCurrentNode(VariableValueModel output, Date eventTime) {
         NodeRunModel crn = getCurrentNodeRun();
         crn.status = LHStatus.COMPLETED;
@@ -602,21 +724,25 @@ public void advanceFrom(NodeModel curNode, VariableValueModel output) {
                 }
             } catch (LHVarSubError exn) {
                 log.debug("Failing threadrun due to VarSubError {} {}", wfRun.getId(), currentNodePosition, exn);
-                fail(
-                        new FailureModel(
-                                "Failed evaluating outgoing edge: " + exn.getMessage(), LHConstants.VAR_MUTATION_ERROR),
-                        new Date());
+                getCurrentNodeRun()
+                        .fail(
+                                new FailureModel(
+                                        "Failed evaluating outgoing edge: " + exn.getMessage(),
+                                        LHConstants.VAR_MUTATION_ERROR),
+                                new Date());
                 return;
             }
         }
         if (nextNode == null) {
-            // TODO: Later versions should validate wfSpec's so that this is not
-            // possible
-            fail(
-                    new FailureModel(
-                            "WfSpec was invalid. There were no activated outgoing edges" + " from a non-exit node.",
-                            LHConstants.INTERNAL_ERROR),
-                    new Date());
+            // TODO: Later versions should validate wfSpec's so that this is not possible; however, it may
+            // always require some runtime checks.
+            getCurrentNodeRun()
+                    .fail(
+                            new FailureModel(
+                                    "WfSpec was invalid. There were no activated outgoing edges"
+                                            + " from a non-exit node.",
+                                    LHConstants.INTERNAL_ERROR),
+                            new Date());
         } else {
             activateNode(nextNode);
         }
@@ -841,25 +967,6 @@ private void applyVarMutationOnAppropriateThread(String varName, VariableMutator
         }
     }
 
-    /**
-     * Creates a new variable on the current ThreadRun or any of its parents
-     * depending on who has
-     * the variable on its definition
-     *
-     * @param varName name of the variable
-     * @param var     value of the variable
-     * @throws LHVarSubError when the varName is not found either on the current
-     *                       ThreadRun
-     *                       definition or its parents definition
-     */
-    public void createVariable(String varName, VariableValueModel var) throws LHVarSubError {
-        VariableMutator createVariable = (wfRunId, threadRunNumber, wfRun) -> {
-            VariableModel variable = new VariableModel(varName, var, wfRunId, threadRunNumber, wfRun.getWfSpec());
-            processorContext.getableManager().put(variable);
-        };
-        applyVarMutationOnAppropriateThread(varName, createVariable);
-    }
-
     /**
      * Mutates an existing variable on the current ThreadRun or any of its parents
      * depending on who
@@ -897,6 +1004,7 @@ public VariableModel getVariable(String varName) {
                 .getAllVariables()
                 .get(varName);
         if (threadVarDef.getAccessLevel() == WfRunVariableAccessLevel.INHERITED_VAR) {
+            log.warn("{}", varName);
             // If we validate the WfSpec properly, it should be impossible for parentWfRunId to be null.
             WfRunIdModel parentWfRunId = getWfRun().getId().getParentWfRunId();
             WfRunModel parentWfRun = processorContext.getableManager().get(parentWfRunId);
diff --git a/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/WfRunModel.java b/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/WfRunModel.java
index 0fe7119b9..0d1c6a6bf 100644
--- a/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/WfRunModel.java
+++ b/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/WfRunModel.java
@@ -6,7 +6,6 @@
 import io.littlehorse.common.LHSerializable;
 import io.littlehorse.common.exceptions.LHApiException;
 import io.littlehorse.common.exceptions.LHValidationError;
-import io.littlehorse.common.exceptions.LHVarSubError;
 import io.littlehorse.common.model.AbstractGetable;
 import io.littlehorse.common.model.CoreGetable;
 import io.littlehorse.common.model.LHTimer;
@@ -25,8 +24,6 @@
 import io.littlehorse.common.model.getable.global.wfspec.WfSpecModel;
 import io.littlehorse.common.model.getable.global.wfspec.WorkflowRetentionPolicyModel;
 import io.littlehorse.common.model.getable.global.wfspec.thread.ThreadSpecModel;
-import io.littlehorse.common.model.getable.global.wfspec.thread.ThreadVarDefModel;
-import io.littlehorse.common.model.getable.global.wfspec.variable.VariableDefModel;
 import io.littlehorse.common.model.getable.objectId.WfRunIdModel;
 import io.littlehorse.common.model.getable.objectId.WfSpecIdModel;
 import io.littlehorse.common.proto.TagStorageType;
@@ -38,7 +35,6 @@
 import io.littlehorse.sdk.common.proto.ThreadRun;
 import io.littlehorse.sdk.common.proto.ThreadType;
 import io.littlehorse.sdk.common.proto.WfRun;
-import io.littlehorse.sdk.common.proto.WfRunVariableAccessLevel;
 import io.littlehorse.sdk.common.proto.WfSpecId;
 import io.littlehorse.server.streams.storeinternals.GetableIndex;
 import io.littlehorse.server.streams.storeinternals.index.IndexedField;
@@ -263,60 +259,14 @@ public ThreadRunModel startThread(
             thread.number = this.greatestThreadRunNumber;
         }
 
-        thread.status = LHStatus.RUNNING;
         thread.threadSpecName = threadName;
         thread.currentNodePosition = -1; // this gets bumped when we start the thread
 
-        thread.startTime = new Date();
-
         thread.wfRun = this;
         thread.type = type;
         threadRuns.add(thread);
 
-        try {
-            tspec.validateStartVariables(variables);
-        } catch (LHValidationError exn) {
-            log.error("Invalid variables received", exn);
-            // TODO: determine how observability events should look like for this case.
-            thread.fail(
-                    new FailureModel(
-                            "Failed validating variables on start: " + exn.getMessage(),
-                            LHConstants.VAR_MUTATION_ERROR),
-                    thread.startTime);
-            return thread;
-        }
-
-        for (ThreadVarDefModel threadVarDef : tspec.getVariableDefs()) {
-            VariableDefModel varDef = threadVarDef.getVarDef();
-            String varName = varDef.getName();
-            VariableValueModel val;
-
-            if (threadVarDef.getAccessLevel() == WfRunVariableAccessLevel.INHERITED_VAR) {
-                if (variables.containsKey(varName)) {
-                    // TODO: handle exception and fail the request.
-                }
-
-                // We do NOT create a variable since we want to use the one from the parent.
-                continue;
-            }
-
-            if (variables.containsKey(varName)) {
-                val = variables.get(varName);
-            } else if (varDef.getDefaultValue() != null) {
-                val = varDef.getDefaultValue();
-            } else {
-                // TODO: Will need to update this when we add the required variable feature.
-                val = new VariableValueModel();
-            }
-
-            try {
-                thread.createVariable(varName, val);
-            } catch (LHVarSubError exn) {
-                throw new RuntimeException("Not possible");
-            }
-        }
-        thread.activateNode(thread.getCurrentNode());
-        thread.advance(start);
+        thread.validateVariablesAndStart(variables);
         return thread;
     }
 
@@ -399,15 +349,12 @@ private boolean startXnHandlers(Date time) {
             }
             somethingChanged = true;
             pendingFailures.remove(i);
-            Map vars;
+            Map vars = new HashMap<>();
 
             ThreadSpecModel iSpec = wfSpec.threadSpecs.get(pfh.handlerSpecName);
             if (iSpec.variableDefs.size() > 0) {
-                vars = new HashMap<>();
                 FailureModel failure = failedThr.getCurrentNodeRun().getLatestFailure();
                 vars.put(LHConstants.EXT_EVT_HANDLER_VAR, failure.content);
-            } else {
-                vars = new HashMap<>();
             }
 
             ThreadRunModel fh =
diff --git a/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/subnoderun/ExitRunModel.java b/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/subnoderun/ExitRunModel.java
index 84cb15484..b3e3aa9fa 100644
--- a/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/subnoderun/ExitRunModel.java
+++ b/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/subnoderun/ExitRunModel.java
@@ -86,11 +86,9 @@ public void arrive(Date time) {
                 nodeRun.fail(getNode().exitNode.failureDef.getFailure(nodeRun.getThreadRun()), time);
             }
         } else {
-            nodeRun.getThreadRun()
-                    .fail(
-                            new FailureModel(
-                                    "Child thread (or threads) failed:" + failedChildren, LHConstants.CHILD_FAILURE),
-                            time);
+            nodeRun.fail(
+                    new FailureModel("Child thread (or threads) failed:" + failedChildren, LHConstants.CHILD_FAILURE),
+                    time);
         }
     }
 }
diff --git a/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/subnoderun/StartMultipleThreadsRunModel.java b/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/subnoderun/StartMultipleThreadsRunModel.java
index a89d99f72..08da50c06 100644
--- a/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/subnoderun/StartMultipleThreadsRunModel.java
+++ b/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/subnoderun/StartMultipleThreadsRunModel.java
@@ -56,7 +56,6 @@ public Class getProtoBaseClass() {
 
     @Override
     public boolean advanceIfPossible(Date time) {
-        log.warn("Shouldn't get here");
         return false;
     }
 
diff --git a/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/subnoderun/WaitForThreadsRunModel.java b/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/subnoderun/WaitForThreadsRunModel.java
index 4e0a225ae..7c1d52669 100644
--- a/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/subnoderun/WaitForThreadsRunModel.java
+++ b/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/subnoderun/WaitForThreadsRunModel.java
@@ -11,27 +11,38 @@
 import io.littlehorse.common.model.getable.core.wfrun.WfRunModel;
 import io.littlehorse.common.model.getable.core.wfrun.failure.FailureModel;
 import io.littlehorse.common.model.getable.core.wfrun.subnoderun.utils.WaitForThreadModel;
+import io.littlehorse.common.model.getable.global.wfspec.WfSpecModel;
+import io.littlehorse.common.model.getable.global.wfspec.node.FailureHandlerDefModel;
 import io.littlehorse.common.model.getable.global.wfspec.node.subnode.WaitForThreadsNodeModel;
+import io.littlehorse.common.model.getable.global.wfspec.thread.ThreadSpecModel;
 import io.littlehorse.sdk.common.proto.LHStatus;
-import io.littlehorse.sdk.common.proto.WaitForThreadsPolicy;
+import io.littlehorse.sdk.common.proto.ThreadType;
 import io.littlehorse.sdk.common.proto.WaitForThreadsRun;
+import io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitingThreadStatus;
+import io.littlehorse.sdk.wfsdk.WorkflowThread;
 import io.littlehorse.server.streams.topology.core.ExecutionContext;
+import io.littlehorse.server.streams.topology.core.ProcessorExecutionContext;
 import java.util.ArrayList;
 import java.util.Date;
+import java.util.HashMap;
 import java.util.List;
-import java.util.Objects;
+import java.util.Map;
 import lombok.Getter;
 import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
 
 @Getter
 @Setter
+@Slf4j
 public class WaitForThreadsRunModel extends SubNodeRun {
 
-    private List threads;
-    private WaitForThreadsPolicy policy;
+    private List threads = new ArrayList<>();
+    private ProcessorExecutionContext context;
 
-    public WaitForThreadsRunModel() {
-        this.threads = new ArrayList<>();
+    public WaitForThreadsRunModel() {}
+
+    public WaitForThreadsRunModel(ProcessorExecutionContext context) {
+        this.context = context;
     }
 
     public Class getProtoBaseClass() {
@@ -40,11 +51,11 @@ public Class getProtoBaseClass() {
 
     @Override
     public void initFrom(Message proto, ExecutionContext context) {
+        this.context = context.castOnSupport(ProcessorExecutionContext.class);
         WaitForThreadsRun p = (WaitForThreadsRun) proto;
         for (WaitForThreadsRun.WaitForThread wft : p.getThreadsList()) {
             threads.add(LHSerializable.fromProto(wft, WaitForThreadModel.class, context));
         }
-        policy = p.getPolicy();
     }
 
     public WaitForThreadsRun.Builder toProto() {
@@ -53,7 +64,6 @@ public WaitForThreadsRun.Builder toProto() {
         for (WaitForThreadModel wft : threads) {
             out.addThreads(wft.toProto());
         }
-        out.setPolicy(policy);
         return out;
     }
 
@@ -70,16 +80,17 @@ public boolean canBeInterrupted() {
 
     @Override
     public boolean advanceIfPossible(Date time) {
-        NodeTerminationManager nodeTerminationManager = new NodeTerminationManager(threads, nodeRun);
-        return nodeTerminationManager.completeIfPossible(time);
+        NodeTerminationManager nodeTerminationManager = new NodeTerminationManager(threads, nodeRun, context);
+        return nodeTerminationManager.advanceIfPossible(time);
     }
 
+    @Override
     public void arrive(Date time) {
         // Need to initialize all of the threads.
         WaitForThreadsNodeModel wftn = getNode().getWaitForThreadsNode();
         nodeRun.setStatus(LHStatus.RUNNING);
         try {
-            threads.addAll(wftn.getThreadsToWaitFor(nodeRun));
+            threads.addAll(wftn.getThreadsToWaitFor(nodeRun, time));
         } catch (LHVarSubError exn) {
             nodeRun.fail(
                     new FailureModel(
@@ -90,83 +101,178 @@ public void arrive(Date time) {
     }
 
     static final class NodeTerminationManager {
-        private final List waitingThreads;
+        private final List childrenToWaitFor;
         private final NodeRunModel waitForThreadsNodeRun;
         private final WfRunModel wfRun;
+        private final ProcessorExecutionContext context;
 
-        NodeTerminationManager(List waitingThreads, NodeRunModel waitForThreadsNodeRun) {
-            this.waitingThreads = waitingThreads;
+        NodeTerminationManager(
+                List waitingThreads,
+                NodeRunModel waitForThreadsNodeRun,
+                ProcessorExecutionContext context) {
+            this.childrenToWaitFor = waitingThreads;
             this.waitForThreadsNodeRun = waitForThreadsNodeRun;
             this.wfRun = waitForThreadsNodeRun.getThreadRun().getWfRun();
+            this.context = context;
         }
 
         /**
          * Verify if WaitForThreads Node should complete or fail.
          * @param time WfRun execution time
-         * @return true if node should advance
+         * @return true if something changed (only God knows how WfRunModel#advance() works)
          */
-        public boolean completeIfPossible(Date time) {
-            WaitForThreadModel failedWaitingThread = getFailedWaitingThread(time);
-            if (failedWaitingThread != null) {
-                handleWaitingThreadFailure(failedWaitingThread, time);
-                // node fails, so advance
+        public boolean advanceIfPossible(Date time) {
+            WaitForThreadModel failedChildThread = getFailedChildThreadIfExists(time);
+            if (failedChildThread != null) {
+                log.trace("Detected child thread failure {}", failedChildThread);
+                handleChildThreadFailure(failedChildThread, time);
                 return true;
             }
-            if (areWaitingThreadsTerminated()) {
+            if (areChildThreadsTerminated()) {
                 waitForThreadsNodeRun.complete(new VariableValueModel(), time);
             }
             return false;
         }
 
-        private void handleWaitingThreadFailure(WaitForThreadModel failedWaitingThread, Date time) {
+        private void handleChildThreadFailure(WaitForThreadModel failedWaitingThread, Date time) {
             ThreadRunModel threadRun = wfRun.getThreadRun(failedWaitingThread.getThreadRunNumber());
-            FailureModel latestFailure =
-                    threadRun.getNodeRun(threadRun.getCurrentNodePosition()).getLatestFailure();
-            if (latestFailure.isUserDefinedFailure()) {
-                propagateFailure(latestFailure, time);
+            FailureModel latestFailure = threadRun.getCurrentNodeRun().getLatestFailure();
+
+            FailureHandlerDefModel handler = getHandlerFor(latestFailure);
+            if (handler != null) {
+                startFailureHandlerFor(handler, latestFailure, threadRun, failedWaitingThread);
+            } else if (latestFailure.isUserDefinedFailure()) {
+                propagateFailure(failedWaitingThread, latestFailure, time);
             } else {
                 String failureMessage =
                         "Some child threads failed = [%s]".formatted(failedWaitingThread.getThreadRunNumber());
                 FailureModel childFailure = new FailureModel(failureMessage, LHConstants.CHILD_FAILURE);
-                propagateFailure(childFailure, time);
+                propagateFailure(failedWaitingThread, childFailure, time);
             }
-            updateWaitingThreadStatuses(failedWaitingThread);
+            updateWaitingThreadStatuses();
         }
 
-        private WaitForThreadModel getFailedWaitingThread(Date time) {
-            for (WaitForThreadModel waitingThread : waitingThreads) {
-                ThreadRunModel threadRun = wfRun.getThreadRun(waitingThread.getThreadRunNumber());
-                waitingThread.setThreadStatus(threadRun.getStatus());
-                if (waitingThread.isFailed() && !waitingThread.isAlreadyHandled()) {
-                    waitingThread.setAlreadyHandled(true);
-                    waitingThread.setThreadEndTime(time);
-                    return waitingThread;
+        /*
+         * If the associated WaitForThreadsRun for this WaitForThreadsRunModel is waiting for a ThreadRun
+         * that has *already* failed, it returns the first one that is failed.
+         */
+        private WaitForThreadModel getFailedChildThreadIfExists(Date time) {
+            for (WaitForThreadModel childThread : childrenToWaitFor) {
+                ThreadRunModel childThreadRun = wfRun.getThreadRun(childThread.getThreadRunNumber());
+                childThread.setThreadStatus(childThreadRun.getStatus());
+
+                // The Waiting Thread starts out in the THREAD_IN_PROGRESS status.
+                if (childThread.isFailed()
+                        && (childThread.getWaitingStatus() == WaitingThreadStatus.THREAD_IN_PROGRESS)) {
+                    childThread.setThreadEndTime(time);
+                    return childThread;
                 }
             }
             return null;
         }
 
-        private boolean areWaitingThreadsTerminated() {
-            return waitingThreads.stream()
-                    .map(WaitForThreadModel::getThreadStatus)
-                    .allMatch(this::isTerminated);
+        private void startFailureHandlerFor(
+                FailureHandlerDefModel failureHandlerDef,
+                FailureModel failure,
+                ThreadRunModel failedChildThreadRun,
+                WaitForThreadModel failedChildThread) {
+            /*
+             * The logic for the ThreadRunModel starting a FailureHandler is more complex
+             * than the logic for the WaitForThreadsRunModel starting a FailureHandler.
+             *
+             * This is because of two things:
+             * 1. In this class, we know the failed ThreadRun is already terminated, so its
+             *    children are also terminated.
+             * 2. We don't need to resurrect the failed ThreadRun after the Failure Handler
+             *    completes; so we don't need to play with the ThreadHaltReasons.
+             */
+
+            log.trace("Starting failure handler for failed child threadrun {}", failedChildThread);
+
+            failedChildThread.setWaitingStatus(WaitingThreadStatus.THREAD_HANDLING_FAILURE);
+
+            WfSpecModel wfSpec = context.service().getWfSpec(waitForThreadsNodeRun.getWfSpecId());
+            ThreadSpecModel handlerSpec = wfSpec.threadSpecs.get(failureHandlerDef.getHandlerSpecName());
+            Map vars = new HashMap<>();
+            if (handlerSpec.variableDefs.size() > 0) {
+                vars.put(WorkflowThread.HANDLER_INPUT_VAR, failure.getContent());
+            }
+            ThreadRunModel failureHandler = wfRun.startThread(
+                    failureHandlerDef.getHandlerSpecName(),
+                    new Date(),
+                    failedChildThreadRun.getNumber(),
+                    vars,
+                    ThreadType.FAILURE_HANDLER);
+            failedChildThread.setFailureHandlerThreadRunId(failureHandler.getNumber());
+        }
+
+        /*
+         * The `WaitForThreadsNode` has a `repeated FailureHandlerDef per_thread_failure_handlers` field,
+         * which defines `FailureHandler`s to run in case a `ThreadRun` that we are waiting for is failed.
+         *
+         * This function returns the first matching FailureHandlerDef if any exists; else it returns null.
+         */
+        private FailureHandlerDefModel getHandlerFor(FailureModel failure) {
+            for (FailureHandlerDefModel handler :
+                    waitForThreadsNodeRun.getNode().getWaitForThreadsNode().getPerThreadFailureHandlers()) {
+                if (handler.doesHandle(failure.getFailureName())) {
+                    return handler;
+                }
+            }
+            return null;
         }
 
-        private boolean isTerminated(LHStatus threadStatus) {
-            return threadStatus == LHStatus.COMPLETED
-                    || threadStatus == LHStatus.ERROR
-                    || threadStatus == LHStatus.EXCEPTION;
+        private boolean areChildThreadsTerminated() {
+            updateWaitingThreadStatuses();
+            return childrenToWaitFor.stream()
+                    .allMatch(childThread ->
+                            childThread.getWaitingStatus() == WaitingThreadStatus.THREAD_COMPLETED_OR_FAILURE_HANDLED);
         }
 
-        private void propagateFailure(FailureModel failureToPropagate, Date time) {
+        private void propagateFailure(
+                WaitForThreadModel failedWaitingThread, FailureModel failureToPropagate, Date time) {
+            failedWaitingThread.setWaitingStatus(WaitingThreadStatus.THREAD_UNSUCCESSFUL);
             waitForThreadsNodeRun.fail(failureToPropagate, time);
         }
 
-        private void updateWaitingThreadStatuses(WaitForThreadModel failedWaitingThread) {
-            for (WaitForThreadModel waitingThread : waitingThreads) {
-                if (!Objects.equals(failedWaitingThread, waitingThread)) {
-                    ThreadRunModel waitingThreadRun = wfRun.getThreadRun(waitingThread.getThreadRunNumber());
-                    waitingThread.setThreadStatus(waitingThreadRun.getStatus());
+        private void updateWaitingThreadStatuses() {
+            for (WaitForThreadModel childThread : childrenToWaitFor) {
+                ThreadRunModel childThreadRun = wfRun.getThreadRun(childThread.getThreadRunNumber());
+                childThread.setThreadStatus(childThreadRun.getStatus());
+
+                if (childThreadRun.getStatus() == LHStatus.COMPLETED) {
+                    childThread.setWaitingStatus(WaitingThreadStatus.THREAD_COMPLETED_OR_FAILURE_HANDLED);
+
+                } else if (childThread.getWaitingStatus() == WaitingThreadStatus.THREAD_HANDLING_FAILURE) {
+                    // Get the failure handler.
+                    Integer failureHandlerId = childThread.getFailureHandlerThreadRunId();
+
+                    if (failureHandlerId == null) {
+                        log.trace("still waiting for failure handler thread to start on waitforthreadsnode");
+                        continue;
+                    }
+
+                    ThreadRunModel failureHandlerThread = wfRun.getThreadRun(failureHandlerId);
+                    if (failureHandlerThread.getStatus() == LHStatus.COMPLETED) {
+                        childThread.setWaitingStatus(WaitingThreadStatus.THREAD_COMPLETED_OR_FAILURE_HANDLED);
+                        // TODO: this is duplicate logic from ThreadRunModel. We need to note that the
+                        // child ThreadRun has already been handled.
+                        waitForThreadsNodeRun
+                                .getThreadRun()
+                                .getHandledFailedChildren()
+                                .add(childThread.getThreadRunNumber());
+
+                    } else if (failureHandlerThread.getStatus() == LHStatus.EXCEPTION) {
+                        propagateFailure(
+                                childThread,
+                                failureHandlerThread.getCurrentNodeRun().getLatestFailure(),
+                                new Date());
+                    } else if (failureHandlerThread.getStatus() == LHStatus.ERROR) {
+                        String failureMessage =
+                                "Failure handler thread failed: [%s]".formatted(failureHandlerThread.getNumber());
+                        FailureModel childFailure = new FailureModel(failureMessage, LHConstants.CHILD_FAILURE);
+                        propagateFailure(childThread, childFailure, new Date());
+                    }
                 }
             }
         }
diff --git a/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/subnoderun/utils/WaitForThreadModel.java b/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/subnoderun/utils/WaitForThreadModel.java
index 495d346b0..679bd1d48 100644
--- a/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/subnoderun/utils/WaitForThreadModel.java
+++ b/server/src/main/java/io/littlehorse/common/model/getable/core/wfrun/subnoderun/utils/WaitForThreadModel.java
@@ -4,13 +4,13 @@
 import io.littlehorse.common.LHConstants;
 import io.littlehorse.common.LHSerializable;
 import io.littlehorse.common.exceptions.LHVarSubError;
-import io.littlehorse.common.model.corecommand.CommandModel;
 import io.littlehorse.common.model.getable.core.noderun.NodeRunModel;
 import io.littlehorse.common.model.getable.core.wfrun.ThreadRunModel;
 import io.littlehorse.common.model.getable.core.wfrun.failure.FailureModel;
 import io.littlehorse.common.util.LHUtil;
 import io.littlehorse.sdk.common.proto.LHStatus;
 import io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitForThread;
+import io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitingThreadStatus;
 import io.littlehorse.server.streams.topology.core.ExecutionContext;
 import java.util.Date;
 import lombok.Getter;
@@ -23,7 +23,8 @@ public class WaitForThreadModel extends LHSerializable {
     private Date threadEndTime;
     private LHStatus threadStatus;
     private int threadRunNumber;
-    private boolean alreadyHandled;
+    private WaitingThreadStatus waitingStatus;
+    private Integer failureHandlerThreadRunId;
     private ExecutionContext executionContext;
 
     public Class getProtoBaseClass() {
@@ -33,7 +34,7 @@ public Class getProtoBaseClass() {
     public WaitForThreadModel() {}
 
     public WaitForThreadModel(
-            NodeRunModel waitForThreadNodeRunModel, Integer threadRunNumberToWaitFor, CommandModel currentCommand)
+            NodeRunModel waitForThreadNodeRunModel, Integer threadRunNumberToWaitFor, Date currentCommandTime)
             throws LHVarSubError {
         ThreadRunModel parentThreadRunModel = waitForThreadNodeRunModel.getThreadRun();
         this.threadRunNumber = threadRunNumberToWaitFor;
@@ -51,12 +52,13 @@ public WaitForThreadModel(
                         new FailureModel(
                                 "Determined threadrunnumber " + threadRunNumber + " is a parent!",
                                 LHConstants.VAR_SUB_ERROR),
-                        currentCommand.getTime());
+                        currentCommandTime);
             }
             potentialParent = potentialParent.getParent();
         }
 
         this.threadStatus = threadRunModel.getStatus();
+        this.waitingStatus = WaitingThreadStatus.THREAD_IN_PROGRESS;
     }
 
     public void initFrom(Message proto, ExecutionContext context) {
@@ -66,7 +68,11 @@ public void initFrom(Message proto, ExecutionContext context) {
         }
         threadStatus = p.getThreadStatus();
         threadRunNumber = p.getThreadRunNumber();
-        alreadyHandled = p.getAlreadyHandled();
+        waitingStatus = p.getWaitingStatus();
+
+        if (p.hasFailureHandlerThreadRunId()) {
+            failureHandlerThreadRunId = p.getFailureHandlerThreadRunId();
+        }
     }
 
     public WaitForThread.Builder toProto() {
@@ -75,7 +81,9 @@ public WaitForThread.Builder toProto() {
         if (threadEndTime != null) {
             out.setThreadEndTime(LHUtil.fromDate(threadEndTime));
         }
-        out.setAlreadyHandled(alreadyHandled);
+        out.setWaitingStatus(waitingStatus);
+
+        if (failureHandlerThreadRunId != null) out.setFailureHandlerThreadRunId(failureHandlerThreadRunId);
         return out;
     }
 
diff --git a/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/node/FailureHandlerDefModel.java b/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/node/FailureHandlerDefModel.java
index 496a34ddc..962e1e095 100644
--- a/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/node/FailureHandlerDefModel.java
+++ b/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/node/FailureHandlerDefModel.java
@@ -6,9 +6,11 @@
 import io.littlehorse.sdk.common.proto.FailureHandlerDef;
 import io.littlehorse.sdk.common.proto.FailureHandlerDef.LHFailureType;
 import io.littlehorse.server.streams.topology.core.ExecutionContext;
+import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
+@Getter
 public class FailureHandlerDefModel extends LHSerializable {
 
     public String specificFailure;
diff --git a/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/node/NodeModel.java b/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/node/NodeModel.java
index 19dc873e9..d5f4281a4 100644
--- a/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/node/NodeModel.java
+++ b/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/node/NodeModel.java
@@ -212,12 +212,7 @@ public void validate() throws LHApiException {
             }
         }
 
-        try {
-            getSubNode().validate();
-        } catch (LHApiException exn) {
-            // Decorate the exception with contextual info
-            throw exn.getCopyWithPrefix("Sub Node");
-        }
+        getSubNode().validate();
     }
 
     public SubNode getSubNode() {
diff --git a/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/node/ThreadsToWaitForModel.java b/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/node/ThreadsToWaitForModel.java
new file mode 100644
index 000000000..3799aaf89
--- /dev/null
+++ b/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/node/ThreadsToWaitForModel.java
@@ -0,0 +1,39 @@
+package io.littlehorse.common.model.getable.global.wfspec.node;
+
+import com.google.protobuf.Message;
+import io.littlehorse.common.LHSerializable;
+import io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor;
+import io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitFor;
+import io.littlehorse.server.streams.topology.core.ExecutionContext;
+import java.util.ArrayList;
+import java.util.List;
+import lombok.Getter;
+
+@Getter
+public class ThreadsToWaitForModel extends LHSerializable {
+
+    private List threads;
+
+    public ThreadsToWaitForModel() {
+        threads = new ArrayList<>();
+    }
+
+    public Class getProtoBaseClass() {
+        return ThreadsToWaitFor.class;
+    }
+
+    public ThreadsToWaitFor.Builder toProto() {
+        ThreadsToWaitFor.Builder out = ThreadsToWaitFor.newBuilder();
+        for (ThreadToWaitForModel thread : threads) {
+            out.addThreads(thread.toProto());
+        }
+        return out;
+    }
+
+    public void initFrom(Message proto, ExecutionContext ctx) {
+        ThreadsToWaitFor p = (ThreadsToWaitFor) proto;
+        for (ThreadToWaitFor thread : p.getThreadsList()) {
+            threads.add(LHSerializable.fromProto(thread, ThreadToWaitForModel.class, ctx));
+        }
+    }
+}
diff --git a/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/node/subnode/WaitForThreadsNodeModel.java b/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/node/subnode/WaitForThreadsNodeModel.java
index ac9caf70d..b10ca0751 100644
--- a/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/node/subnode/WaitForThreadsNodeModel.java
+++ b/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/node/subnode/WaitForThreadsNodeModel.java
@@ -10,13 +10,15 @@
 import io.littlehorse.common.model.getable.core.wfrun.ThreadRunModel;
 import io.littlehorse.common.model.getable.core.wfrun.subnoderun.WaitForThreadsRunModel;
 import io.littlehorse.common.model.getable.core.wfrun.subnoderun.utils.WaitForThreadModel;
+import io.littlehorse.common.model.getable.global.wfspec.node.FailureHandlerDefModel;
 import io.littlehorse.common.model.getable.global.wfspec.node.SubNode;
 import io.littlehorse.common.model.getable.global.wfspec.node.ThreadToWaitForModel;
+import io.littlehorse.common.model.getable.global.wfspec.node.ThreadsToWaitForModel;
 import io.littlehorse.common.model.getable.global.wfspec.variable.VariableAssignmentModel;
+import io.littlehorse.sdk.common.proto.FailureHandlerDef;
 import io.littlehorse.sdk.common.proto.VariableType;
 import io.littlehorse.sdk.common.proto.WaitForThreadsNode;
-import io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadToWaitFor;
-import io.littlehorse.sdk.common.proto.WaitForThreadsPolicy;
+import io.littlehorse.sdk.common.proto.WaitForThreadsNode.ThreadsToWaitForCase;
 import io.littlehorse.server.streams.topology.core.ExecutionContext;
 import io.littlehorse.server.streams.topology.core.ProcessorExecutionContext;
 import java.util.ArrayList;
@@ -26,87 +28,118 @@
 import java.util.Set;
 import lombok.Getter;
 import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
 
+@Slf4j
 @Getter
 @Setter
 public class WaitForThreadsNodeModel extends SubNode {
 
-    private List threads;
-
-    private WaitForThreadsPolicy policy;
+    private ExecutionContext context;
 
+    private ThreadsToWaitForCase type;
+    private ThreadsToWaitForModel threads;
     private VariableAssignmentModel threadList;
-    private ExecutionContext context;
-    private ProcessorExecutionContext processorContext;
+
+    private List perThreadFailureHandlers;
 
     public Class getProtoBaseClass() {
         return WaitForThreadsNode.class;
     }
 
     public WaitForThreadsNodeModel() {
-        threads = new ArrayList<>();
+        perThreadFailureHandlers = new ArrayList<>();
     }
 
     public void initFrom(Message proto, ExecutionContext context) {
+        this.context = context;
+
         WaitForThreadsNode p = (WaitForThreadsNode) proto;
-        for (ThreadToWaitFor ttwf : p.getThreadsList()) {
-            threads.add(LHSerializable.fromProto(ttwf, ThreadToWaitForModel.class, context));
+        type = p.getThreadsToWaitForCase();
+
+        switch (type) {
+            case THREADS:
+                threads = LHSerializable.fromProto(p.getThreads(), ThreadsToWaitForModel.class, context);
+                break;
+            case THREAD_LIST:
+                threadList = VariableAssignmentModel.fromProto(p.getThreadList(), context);
+                break;
+            case THREADSTOWAITFOR_NOT_SET:
+                log.warn("should be impossible to get unset threadsToWaitFor");
         }
-        policy = p.getPolicy();
-        if (p.hasThreadList()) {
-            threadList = VariableAssignmentModel.fromProto(p.getThreadList(), context);
+
+        for (FailureHandlerDef handler : p.getPerThreadFailureHandlersList()) {
+            perThreadFailureHandlers.add(FailureHandlerDefModel.fromProto(handler, context));
         }
-        this.context = context;
-        this.processorContext = context.castOnSupport(ProcessorExecutionContext.class);
     }
 
     public WaitForThreadsNode.Builder toProto() {
         WaitForThreadsNode.Builder out = WaitForThreadsNode.newBuilder();
-        for (ThreadToWaitForModel ttwf : threads) {
-            out.addThreads(ttwf.toProto());
+        switch (type) {
+            case THREADS:
+                out.setThreads(threads.toProto());
+                break;
+            case THREAD_LIST:
+                out.setThreadList(threadList.toProto());
+                break;
+            case THREADSTOWAITFOR_NOT_SET:
+                log.warn("should be impossible to get unset threadsToWaitFor");
         }
-        out.setPolicy(policy);
-        if (threadList != null) {
-            out.setThreadList(threadList.toProto());
+
+        for (FailureHandlerDefModel handler : perThreadFailureHandlers) {
+            out.addPerThreadFailureHandlers(handler.toProto());
         }
         return out;
     }
 
     public WaitForThreadsRunModel createSubNodeRun(Date time) {
-        WaitForThreadsRunModel waitForThreadsRun = new WaitForThreadsRunModel();
-        waitForThreadsRun.setPolicy(getPolicy());
-        return waitForThreadsRun;
+        return new WaitForThreadsRunModel(context.castOnSupport(ProcessorExecutionContext.class));
     }
 
-    public List getThreadsToWaitFor(NodeRunModel nodeRun) throws LHVarSubError {
+    public List getThreadsToWaitFor(NodeRunModel nodeRun, Date currentCommandTime)
+            throws LHVarSubError {
         ThreadRunModel thread = nodeRun.getThreadRun();
         List out = new ArrayList<>();
 
-        for (ThreadToWaitForModel ttwf : getThreads()) {
-            int threadRunNumber = thread.assignVariable(ttwf.getThreadRunNumber())
-                    .asInt()
-                    .getIntVal()
-                    .intValue();
-            out.add(new WaitForThreadModel(nodeRun, threadRunNumber, processorContext.currentCommand()));
+        switch (type) {
+            case THREADS:
+                for (ThreadToWaitForModel ttwf : threads.getThreads()) {
+                    int threadRunNumber = thread.assignVariable(ttwf.getThreadRunNumber())
+                            .asInt()
+                            .getIntVal()
+                            .intValue();
+                    out.add(new WaitForThreadModel(nodeRun, threadRunNumber, currentCommandTime));
+                }
+                break;
+            case THREAD_LIST:
+                VariableValueModel threadListVar = thread.assignVariable(threadList);
+
+                for (Object threadNumberObj : threadListVar.getJsonArrVal()) {
+                    out.add(new WaitForThreadModel(
+                            nodeRun, Integer.valueOf(threadNumberObj.toString()), currentCommandTime));
+                }
+                break;
+            case THREADSTOWAITFOR_NOT_SET:
+                log.warn("Should be impossible to have unset threadsToWaitFor");
         }
-
-        if (threadList != null) {
-            VariableValueModel threadListVar = thread.assignVariable(threadList);
-
-            for (Object threadNumberObj : threadListVar.getJsonArrVal()) {
-                out.add(new WaitForThreadModel(
-                        nodeRun, Integer.valueOf(threadNumberObj.toString()), processorContext.currentCommand()));
-            }
-        }
-
         return out;
     }
 
     @Override
     public Set getNeededVariableNames() {
         Set out = new HashSet<>();
-        for (ThreadToWaitForModel ttwf : threads) {
-            out.addAll(ttwf.getThreadRunNumber().getRequiredWfRunVarNames());
+
+        switch (type) {
+            case THREADS:
+                for (ThreadToWaitForModel ttwf : threads.getThreads()) {
+                    out.addAll(ttwf.getThreadRunNumber().getRequiredWfRunVarNames());
+                }
+                break;
+            case THREAD_LIST:
+                out.addAll(threadList.getRequiredWfRunVarNames());
+                break;
+            case THREADSTOWAITFOR_NOT_SET:
+                log.warn("Should be impossible");
         }
 
         return out;
@@ -114,11 +147,25 @@ public Set getNeededVariableNames() {
 
     @Override
     public void validate() throws LHApiException {
-        for (ThreadToWaitForModel ttwf : threads) {
-            if (!ttwf.getThreadRunNumber().canBeType(VariableType.INT, node.threadSpecModel)) {
-                throw new LHApiException(
-                        Status.INVALID_ARGUMENT, "`threadRunNumber` for WAIT_FOR_THREAD node must resolve to INT!");
-            }
+        switch (type) {
+            case THREADS:
+                for (ThreadToWaitForModel ttwf : threads.getThreads()) {
+                    if (!ttwf.getThreadRunNumber().canBeType(VariableType.INT, node.getThreadSpecModel())) {
+                        throw new LHApiException(
+                                Status.INVALID_ARGUMENT,
+                                "`threadRunNumber` for WAIT_FOR_THREAD node must resolve to INT!");
+                    }
+                }
+                break;
+            case THREAD_LIST:
+                if (!threadList.canBeType(VariableType.JSON_ARR, node.getThreadSpecModel())) {
+                    throw new LHApiException(
+                            Status.INVALID_ARGUMENT,
+                            "`threadRunNumber` for WAIT_FOR_THREAD node must resolve to JSON_ARR!");
+                }
+                break;
+            case THREADSTOWAITFOR_NOT_SET:
+                log.warn("Should be impossible");
         }
     }
 }
diff --git a/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/thread/ThreadSpecModel.java b/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/thread/ThreadSpecModel.java
index 11a0a1a2e..836f33080 100644
--- a/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/thread/ThreadSpecModel.java
+++ b/server/src/main/java/io/littlehorse/common/model/getable/global/wfspec/thread/ThreadSpecModel.java
@@ -17,6 +17,7 @@
 import io.littlehorse.sdk.common.proto.ThreadVarDef;
 import io.littlehorse.sdk.common.proto.VariableAssignment.SourceCase;
 import io.littlehorse.sdk.common.proto.VariableType;
+import io.littlehorse.sdk.common.proto.WfRunVariableAccessLevel;
 import io.littlehorse.server.streams.topology.core.ExecutionContext;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -294,26 +295,37 @@ private void validateExternalEventDefUse() throws LHApiException {
 
     // TODO: check input variables.
     public void validateStartVariables(Map vars) throws LHValidationError {
-        Map required = getInputVariableDefs();
-
-        for (Map.Entry e : required.entrySet()) {
-            VariableValueModel val = vars.get(e.getKey());
-            VariableDefModel varDef = e.getValue().getVarDef();
+        for (Map.Entry e : getInputVariableDefs().entrySet()) {
+            String varName = e.getKey();
+            ThreadVarDefModel threadVarDef = e.getValue();
+            VariableValueModel val = vars.get(varName);
+            VariableDefModel varDef = threadVarDef.getVarDef();
             if (val == null) {
-                log.debug("Variable {} not provided, defaulting to null", e.getKey());
+                if (threadVarDef.isRequired()) {
+                    throw new LHValidationError(
+                            "Must provide required input variable %s of type %s".formatted(varName, varDef.getType()));
+                }
+                log.debug("Variable {} not provided, defaulting to null", varName);
                 continue;
             }
 
             if (val.getType() != varDef.getType() && val.getType() != null) {
                 throw new LHValidationError(
-                        null, "Var " + e.getKey() + " should be " + varDef.getType() + " but is " + val.getType());
+                        "Var " + varName + " should be " + varDef.getType() + " but is " + val.getType());
+            }
+
+            if (threadVarDef.getAccessLevel() == WfRunVariableAccessLevel.INHERITED_VAR) {
+                if (vars.containsKey(varName)) {
+                    throw new LHValidationError(
+                            "Variable %s is an inherited var but it was provided as input".formatted(varName));
+                }
             }
         }
 
         for (Map.Entry e : vars.entrySet()) {
-            if (getVd(e.getKey()) == null) {
-                throw new LHApiException(
-                        Status.INVALID_ARGUMENT, "Var " + e.getKey() + " provided but not needed for thread " + name);
+            String varName = e.getKey();
+            if (getVd(varName) == null) {
+                throw new LHValidationError("Var " + varName + " provided but not needed for thread " + name);
             }
         }
     }
diff --git a/server/src/test/java/e2e/OutOfScopeVarAccessTest.java b/server/src/test/java/e2e/OutOfScopeVarAccessTest.java
new file mode 100644
index 000000000..eda5071eb
--- /dev/null
+++ b/server/src/test/java/e2e/OutOfScopeVarAccessTest.java
@@ -0,0 +1,67 @@
+package e2e;
+
+import io.littlehorse.sdk.common.proto.LHStatus;
+import io.littlehorse.sdk.common.proto.VariableMutationType;
+import io.littlehorse.sdk.common.proto.VariableType;
+import io.littlehorse.sdk.wfsdk.NodeOutput;
+import io.littlehorse.sdk.wfsdk.SpawnedThread;
+import io.littlehorse.sdk.wfsdk.SpawnedThreads;
+import io.littlehorse.sdk.wfsdk.WfRunVariable;
+import io.littlehorse.sdk.wfsdk.Workflow;
+import io.littlehorse.test.LHTest;
+import io.littlehorse.test.LHWorkflow;
+import io.littlehorse.test.WorkflowVerifier;
+import java.util.Map;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+@LHTest
+public class OutOfScopeVarAccessTest {
+
+    private WorkflowVerifier verifier;
+
+    // Need to put it here so that we can access it in an illegal manner
+    private WfRunVariable variableOnlyAccessibleInChild;
+
+    @LHWorkflow("access-out-of-scope-var")
+    private Workflow accessOutOfScopeVarWf;
+
+    @Test
+    void shouldThrowVarSubErrorOnIllegalAccess() {
+        verifier.prepareRun(accessOutOfScopeVarWf)
+                // this WfRun should be terminated on the first RPC since there are no
+                // TaskRun's, ExternalEvents, or User Tasks. Everything is synchronous.
+                //
+                // Therefore, we should NOT do a waitForStatus().
+                .thenVerifyWfRun(wfRun -> {
+                    Assertions.assertThat(wfRun.getStatus()).isEqualTo(LHStatus.ERROR);
+                    Assertions.assertThat(wfRun.getThreadRuns(1).getStatus()).isEqualTo(LHStatus.EXCEPTION);
+                    Assertions.assertThat(wfRun.getThreadRuns(2).getStatus()).isEqualTo(LHStatus.ERROR);
+                })
+                .start();
+    }
+
+    @LHWorkflow("access-out-of-scope-var")
+    public Workflow getAccessOutOfScopeVarWf() {
+        return Workflow.newWorkflow("access-out-of-scope-var", wf -> {
+            SpawnedThread spawnedThread = wf.spawnThread(
+                    child -> {
+                        variableOnlyAccessibleInChild = child.addVariable("out-of-scope-var", VariableType.BOOL);
+                        child.fail("some-failure", "some failure message");
+                    },
+                    "child-thread",
+                    Map.of());
+
+            // In order to "trick" the SDK into creating a WfSpec that accesses an
+            // out-of-scope variable, we have to use a failure handler on a waitForThreads
+            // node.
+            NodeOutput nodeThatWillFail = wf.waitForThreads(SpawnedThreads.of(spawnedThread));
+
+            wf.handleException(nodeThatWillFail, handler -> {
+                // We once had an NPE in which accessing a variable from the child in this thread
+                // was permitted by the SDK but the server threw an orzdash.
+                handler.mutate(variableOnlyAccessibleInChild, VariableMutationType.ASSIGN, true);
+            });
+        });
+    }
+}
diff --git a/server/src/test/java/e2e/SharedVariablesTest.java b/server/src/test/java/e2e/SharedVariablesTest.java
index 12d4e7e13..9b27cf86c 100644
--- a/server/src/test/java/e2e/SharedVariablesTest.java
+++ b/server/src/test/java/e2e/SharedVariablesTest.java
@@ -1,6 +1,10 @@
 package e2e;
 
+import static org.assertj.core.api.Assertions.assertThat;
+
 import io.littlehorse.sdk.common.proto.LHStatus;
+import io.littlehorse.sdk.common.proto.LittleHorseGrpc.LittleHorseBlockingStub;
+import io.littlehorse.sdk.common.proto.VariableId;
 import io.littlehorse.sdk.common.proto.VariableMutationType;
 import io.littlehorse.sdk.common.proto.VariableType;
 import io.littlehorse.sdk.common.proto.WfRunId;
@@ -14,6 +18,7 @@
 import io.littlehorse.test.LHWorkflow;
 import io.littlehorse.test.WorkflowVerifier;
 import java.util.Objects;
+import java.util.UUID;
 import org.assertj.core.api.Assertions;
 import org.junit.jupiter.api.Test;
 
@@ -21,6 +26,7 @@
 public class SharedVariablesTest {
 
     private WorkflowVerifier workflowVerifier;
+    private LittleHorseBlockingStub client;
 
     @LHWorkflow("shared-variables-parent-wf")
     private Workflow parentWf;
@@ -30,10 +36,16 @@ public class SharedVariablesTest {
 
     @Test
     public void shouldResolvePublicVariablesFromParentWf() {
+        String parentWfRunId = UUID.randomUUID().toString();
+        String childWfRunId = UUID.randomUUID().toString();
+
         WfRunId parentWfRun = workflowVerifier
                 .prepareRun(parentWf, Arg.of("input-number", 3))
                 .waitForStatus(LHStatus.COMPLETED)
-                .start(WfRunId.newBuilder().setId("parent-wf-run").build());
+                .thenVerifyVariable(0, "public-variable", variableValue -> {
+                    assertThat(variableValue.getInt()).isEqualTo(6);
+                })
+                .start(WfRunId.newBuilder().setId(parentWfRunId).build());
 
         workflowVerifier
                 .prepareRun(childWf)
@@ -42,9 +54,17 @@ public void shouldResolvePublicVariablesFromParentWf() {
                     Assertions.assertThat(variableValue.getInt()).isEqualTo(12);
                 })
                 .start(WfRunId.newBuilder()
-                        .setId("child-wf-run")
+                        .setId(childWfRunId)
                         .setParentWfRunId(parentWfRun)
                         .build());
+
+        assertThat(client.getVariable(VariableId.newBuilder()
+                                .setWfRunId(parentWfRun)
+                                .setName("public-variable")
+                                .build())
+                        .getValue()
+                        .getInt())
+                .isEqualTo(12);
     }
 
     @LHWorkflow("shared-variables-parent-wf")
@@ -64,6 +84,7 @@ public Workflow buildChildWf() {
         Workflow out = new WorkflowImpl("shared-variables-child-wf", thread -> {
             WfRunVariable publicVariable = thread.addVariable("public-variable", VariableType.INT)
                     .withAccessLevel(WfRunVariableAccessLevel.INHERITED_VAR);
+
             WfRunVariable calculatedValue =
                     thread.addVariable("calculated-value", VariableType.INT).searchable();
             thread.mutate(publicVariable, VariableMutationType.MULTIPLY, 2);
diff --git a/server/src/test/java/e2e/SpawnThreadsForEachWithFailureHandlerTest.java b/server/src/test/java/e2e/SpawnThreadsForEachWithFailureHandlerTest.java
new file mode 100644
index 000000000..e9ce935a4
--- /dev/null
+++ b/server/src/test/java/e2e/SpawnThreadsForEachWithFailureHandlerTest.java
@@ -0,0 +1,305 @@
+package e2e;
+
+import static io.littlehorse.sdk.common.proto.LHStatus.COMPLETED;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import io.littlehorse.sdk.common.proto.Comparator;
+import io.littlehorse.sdk.common.proto.Failure;
+import io.littlehorse.sdk.common.proto.LHErrorType;
+import io.littlehorse.sdk.common.proto.LHStatus;
+import io.littlehorse.sdk.common.proto.NodeRun.NodeTypeCase;
+import io.littlehorse.sdk.common.proto.ThreadType;
+import io.littlehorse.sdk.common.proto.VariableMutationType;
+import io.littlehorse.sdk.common.proto.VariableType;
+import io.littlehorse.sdk.common.proto.WaitForThreadsRun;
+import io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitForThread;
+import io.littlehorse.sdk.common.proto.WaitForThreadsRun.WaitingThreadStatus;
+import io.littlehorse.sdk.common.util.Arg;
+import io.littlehorse.sdk.wfsdk.SpawnedThreads;
+import io.littlehorse.sdk.wfsdk.WaitForThreadsNodeOutput;
+import io.littlehorse.sdk.wfsdk.WfRunVariable;
+import io.littlehorse.sdk.wfsdk.Workflow;
+import io.littlehorse.sdk.wfsdk.WorkflowThread;
+import io.littlehorse.sdk.wfsdk.internal.WorkflowImpl;
+import io.littlehorse.sdk.worker.LHTaskMethod;
+import io.littlehorse.test.LHTest;
+import io.littlehorse.test.LHWorkflow;
+import io.littlehorse.test.WorkflowVerifier;
+import java.util.List;
+import java.util.Map;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+@LHTest
+public class SpawnThreadsForEachWithFailureHandlerTest {
+
+    @LHWorkflow("handle-exception-on-children")
+    private Workflow handleExceptionOnChildren;
+
+    private WorkflowVerifier verifier;
+
+    private static final String SUCCEED = "succeed";
+    private static final String EXCEPTION_BUT_STILL_FAIL = "exception-but-still-fail";
+    private static final String EXCEPTION_BUT_RECOVER = "exception-but-recover";
+    private static final String EXCEPTION_DONT_HANDLE = "exception-dont-handle";
+    private static final String STILL_FAILED = "still-failed";
+
+    @Test
+    void testSpawnThreadForEachHappyPath() {
+        verifier.prepareRun(
+                        handleExceptionOnChildren,
+                        Arg.of("to-spawn", List.of(new MyInput(SUCCEED, false), new MyInput(SUCCEED, true))))
+                .waitForStatus(LHStatus.COMPLETED)
+                .thenVerifyWfRun(wfRun -> {
+                    Assertions.assertThat(wfRun.getThreadRunsCount()).isEqualTo(3);
+                })
+                .thenVerifyAllTaskRuns(1, taskRuns -> {
+                    Assertions.assertThat(taskRuns).isEmpty();
+                })
+                .thenVerifyAllTaskRuns(2, taskRuns -> {
+                    Assertions.assertThat(taskRuns.size()).isEqualTo(1);
+                })
+                .start();
+    }
+
+    @Test
+    void testExceptionPropagatedToParentWhenNoHandler() {
+        verifier.prepareRun(
+                        handleExceptionOnChildren,
+                        Arg.of("to-spawn", List.of(new MyInput(EXCEPTION_DONT_HANDLE, true))))
+                .waitForStatus(LHStatus.EXCEPTION)
+                .thenVerifyWfRun(wfRun -> {
+                    Assertions.assertThat(wfRun.getThreadRunsCount()).isEqualTo(2);
+                })
+                .thenVerifyNodeRun(0, 2, nr -> {
+                    assertThat(nr.getFailuresCount()).isEqualTo(1);
+                    Failure latestFailure = nr.getFailures(0);
+                    assertThat(latestFailure.getFailureName()).isEqualTo(EXCEPTION_DONT_HANDLE);
+                })
+                .start();
+    }
+
+    @Test
+    void testFirstThreadHasHandlerAndSecondDoesnt() {
+        verifier.prepareRun(
+                        handleExceptionOnChildren,
+                        Arg.of(
+                                "to-spawn",
+                                List.of(
+                                        new MyInput(EXCEPTION_BUT_RECOVER, false),
+                                        new MyInput(EXCEPTION_BUT_STILL_FAIL, true))))
+                .waitForThreadRunStatus(1, LHStatus.EXCEPTION)
+                .waitForThreadRunStatus(2, LHStatus.EXCEPTION)
+                .waitForThreadRunStatus(3, LHStatus.COMPLETED)
+                .waitForThreadRunStatus(4, LHStatus.EXCEPTION)
+                .waitForStatus(LHStatus.EXCEPTION)
+                .thenVerifyWfRun(wfRun -> {
+                    Assertions.assertThat(wfRun.getThreadRunsCount()).isEqualTo(5);
+                })
+                .thenVerifyNodeRun(0, 2, nr -> {
+                    assertThat(nr.getFailuresCount()).isEqualTo(1);
+                    Failure latestFailure = nr.getFailures(0);
+                    assertThat(latestFailure.getFailureName()).isEqualTo(STILL_FAILED);
+                })
+                .start();
+    }
+
+    @Test
+    void testFirstThreadSucceedsSecondThreadRecovers() {
+        verifier.prepareRun(
+                        handleExceptionOnChildren,
+                        Arg.of(
+                                "to-spawn",
+                                List.of(new MyInput(EXCEPTION_BUT_RECOVER, false), new MyInput(SUCCEED, true))))
+                .waitForThreadRunStatus(1, LHStatus.EXCEPTION)
+                .waitForThreadRunStatus(2, LHStatus.COMPLETED)
+                .waitForThreadRunStatus(3, LHStatus.COMPLETED)
+                .waitForStatus(LHStatus.COMPLETED)
+                .thenVerifyWfRun(wfRun -> {
+                    Assertions.assertThat(wfRun.getThreadRunsCount()).isEqualTo(4);
+                })
+                .start();
+    }
+
+    @Test
+    void testOtherThreadHaltedWhenFirstFailsWithoutHandler() {
+        verifier.prepareRun(
+                        handleExceptionOnChildren,
+                        Arg.of(
+                                "to-spawn",
+                                List.of(
+                                        // First thread fails immediately (before second thread finishes)
+                                        new MyInput(EXCEPTION_DONT_HANDLE, false),
+                                        // We will verify that second thread goes to HALTING then HALTED.
+                                        // To ensure that it goes to another command, we make it execute the
+                                        // task.
+                                        new MyInput("success", true))))
+                .waitForThreadRunStatus(1, LHStatus.EXCEPTION)
+                .waitForThreadRunStatus(2, LHStatus.HALTED)
+                .waitForStatus(LHStatus.EXCEPTION)
+                .thenVerifyWfRun(wfRun -> {
+                    Assertions.assertThat(wfRun.getThreadRunsCount()).isEqualTo(3);
+
+                    // Ensure that the second thread didn't advance past the taskrun
+                    Assertions.assertThat(wfRun.getThreadRunsList().get(2).getCurrentNodePosition())
+                            .isEqualTo(2);
+                })
+                .thenVerifyNodeRun(2, 2, nr -> {
+                    // Make sure the second thread still executed the task
+                    assertThat(nr.getNodeTypeCase()).isEqualTo(NodeTypeCase.TASK);
+                    assertThat(nr.getStatus()).isEqualByComparingTo(LHStatus.COMPLETED);
+                })
+                .start();
+    }
+
+    // When one thread fails without a handler, then the NodeRun fails and it causes the
+    // other children to be halted.
+    @Test
+    void testFailureHandlerIgnoredIfWaitForThreadsRunAlreadyFailed() {
+        verifier.prepareRun(
+                        handleExceptionOnChildren,
+                        Arg.of(
+                                "to-spawn",
+                                List.of(
+                                        // The first thread will execute a task before failing
+                                        new MyInput(EXCEPTION_BUT_RECOVER, true),
+                                        // The second thread immediately fails. Therefore, the
+                                        // WaitForThreadsRun is failed before there is time to handle
+                                        // the failure of the first thread.
+                                        new MyInput(EXCEPTION_DONT_HANDLE, false))))
+                .waitForStatus(LHStatus.EXCEPTION)
+                .thenVerifyWfRun(wfRun -> {
+                    // Should NOT have an exception handler for the first thread
+                    Assertions.assertThat(wfRun.getThreadRunsCount()).isEqualTo(3);
+                    Assertions.assertThat(wfRun.getThreadRuns(1).getStatus()).isEqualTo(LHStatus.HALTED);
+                })
+                .thenVerifyNodeRun(0, 2, nr -> {
+                    assertThat(nr.getFailuresCount()).isEqualTo(1);
+                    Failure latestFailure = nr.getFailures(0);
+                    assertThat(latestFailure.getFailureName()).isEqualTo(EXCEPTION_DONT_HANDLE);
+                })
+                .start();
+    }
+
+    @Test
+    void testFirstThreadRecoversSecondThreadFinishes() {
+        verifier.prepareRun(
+                        handleExceptionOnChildren,
+                        Arg.of(
+                                "to-spawn",
+                                List.of(
+                                        // First thread fails immediately but exception handler passes
+                                        new MyInput(EXCEPTION_BUT_RECOVER, false),
+                                        // We will verify that second thread goes to HALTING then HALTED.
+                                        // To ensure that it goes to another command, we make it execute the
+                                        // task.
+                                        new MyInput(SUCCEED, true))))
+                .waitForThreadRunStatus(1, LHStatus.EXCEPTION)
+                .waitForThreadRunStatus(2, LHStatus.COMPLETED)
+                .waitForThreadRunStatus(3, COMPLETED)
+                .waitForStatus(LHStatus.COMPLETED)
+                .thenVerifyNodeRun(0, 2, nr -> {
+                    // Make sure the second thread still executed the task
+                    assertThat(nr.getNodeTypeCase()).isEqualTo(NodeTypeCase.WAIT_THREADS);
+
+                    WaitForThreadsRun wftr = nr.getWaitThreads();
+                    WaitForThread oneThatFailed = wftr.getThreads(0);
+                    assertThat(oneThatFailed.getThreadStatus()).isEqualTo(LHStatus.EXCEPTION);
+                    assertThat(oneThatFailed.getWaitingStatus())
+                            .isEqualTo(WaitingThreadStatus.THREAD_COMPLETED_OR_FAILURE_HANDLED);
+                })
+                .thenVerifyWfRun(wfRun -> {
+                    assertThat(wfRun.getThreadRuns(3).getParentThreadId()).isEqualTo(1);
+                    assertThat(wfRun.getThreadRuns(3).getType()).isEqualTo(ThreadType.FAILURE_HANDLER);
+                })
+                .start();
+    }
+
+    @Test
+    void testBothThreadsFinishBeforeArrivingAtNode() {
+        verifier.prepareRun(
+                        handleExceptionOnChildren,
+                        Arg.of(
+                                "to-spawn",
+                                List.of(
+                                        // First thread succeeds immediately
+                                        new MyInput(SUCCEED, false),
+                                        // Second thread succeeds immediately
+                                        new MyInput(SUCCEED, true))))
+                .waitForStatus(LHStatus.COMPLETED)
+                .start();
+    }
+
+    @LHWorkflow("handle-exception-on-children")
+    public Workflow buildHandleExceptionOnChildren() {
+        return new WorkflowImpl("handle-exception-on-children", wf -> {
+            WfRunVariable threadsToSpawn =
+                    wf.addVariable("to-spawn", VariableType.JSON_ARR).required();
+            WfRunVariable emptyJson = wf.addVariable("json-obj", VariableType.JSON_OBJ);
+
+            SpawnedThreads childThreads = wf.spawnThreadForEach(
+                    threadsToSpawn,
+                    "child",
+                    child -> {
+                        WfRunVariable input =
+                                child.addVariable(WorkflowThread.HANDLER_INPUT_VAR, VariableType.JSON_OBJ);
+
+                        // We want to be able to handle two different types of cases:
+                        // 1. The child thread fails immediately (i.e. in the same Command) when it's started
+                        // 2. The child thread fails later during execution (i.e. in a later Command)
+                        //
+                        // Putting in a TaskRun that is optionally executed allows us to test both cases with the
+                        // same WfSpec.
+                        child.doIf(
+                                child.condition(input.jsonPath("$.executeTask"), Comparator.EQUALS, true), ifBody -> {
+                                    ifBody.execute("dummy-task-spawnthreadtest");
+                                });
+
+                        child.doIf(
+                                child.condition(
+                                        input.jsonPath("$.endResult"), Comparator.EQUALS, EXCEPTION_BUT_RECOVER),
+                                ifBody -> {
+                                    ifBody.fail(EXCEPTION_BUT_RECOVER, "failed due to exception");
+                                });
+                        child.doIf(
+                                child.condition(
+                                        input.jsonPath("$.endResult"), Comparator.EQUALS, EXCEPTION_BUT_STILL_FAIL),
+                                ifBody -> {
+                                    ifBody.fail(EXCEPTION_BUT_STILL_FAIL, "failed due to exception");
+                                });
+                        child.doIf(
+                                child.condition(
+                                        input.jsonPath("$.endResult"), Comparator.EQUALS, EXCEPTION_DONT_HANDLE),
+                                ifBody -> {
+                                    ifBody.fail(EXCEPTION_DONT_HANDLE, "failed due to exception");
+                                });
+
+                        child.doIf(
+                                child.condition(input.jsonPath("$.endResult"), Comparator.EQUALS, "error"), ifBody -> {
+                                    // Cause a VarSubError
+                                    ifBody.mutate(
+                                            emptyJson.jsonPath("$.notarealpath"),
+                                            VariableMutationType.MULTIPLY,
+                                            emptyJson.jsonPath("$.asdfasdf"));
+                                });
+                    },
+                    Map.of());
+
+            WaitForThreadsNodeOutput wftn = wf.waitForThreads(childThreads);
+            wftn.handleErrorOnChild(LHErrorType.VAR_SUB_ERROR, handler -> {});
+
+            wftn.handleExceptionOnChild(EXCEPTION_BUT_RECOVER, handler -> {
+                handler.execute("dummy-task-spawnthreadtest");
+            });
+
+            wftn.handleExceptionOnChild(EXCEPTION_BUT_STILL_FAIL, handler -> {
+                handler.fail(STILL_FAILED, "should still fail");
+            });
+        });
+    }
+
+    @LHTaskMethod("dummy-task-spawnthreadtest")
+    public String obiwan() {
+        return "hello there";
+    }
+}
diff --git a/server/src/test/java/e2e/TaskDefLifecycleTest.java b/server/src/test/java/e2e/TaskDefLifecycleTest.java
index d2fa0d131..bf870da7d 100644
--- a/server/src/test/java/e2e/TaskDefLifecycleTest.java
+++ b/server/src/test/java/e2e/TaskDefLifecycleTest.java
@@ -56,7 +56,7 @@ void shouldThrowAlreadyExistWhenTaskDefDifferent() {
     }
 
     @Test
-    void shouldBeAbleToReadTaskDefImmediatelyAfterCreation() throws Exception {
+    void shouldBeAbleToUseTaskDefImmediatelyAfterCreation() throws Exception {
         // Repeat 5 times to make race conditions more likely to appear
         for (int i = 0; i < 5; i++) {
             String taskDefName = UUID.randomUUID().toString();
diff --git a/server/src/test/java/e2e/UserTaskTest.java b/server/src/test/java/e2e/UserTaskTest.java
index a8dc54e12..aebceeda5 100644
--- a/server/src/test/java/e2e/UserTaskTest.java
+++ b/server/src/test/java/e2e/UserTaskTest.java
@@ -79,7 +79,7 @@ public Workflow buildDeadlineReassignmentWorkflow() {
             UserTaskOutput formOutput =
                     entrypointThread.assignUserTask(USER_TASK_DEF_NAME, "test-group", "test-department");
 
-            entrypointThread.releaseToGroupOnDeadline(formOutput, 4);
+            entrypointThread.releaseToGroupOnDeadline(formOutput, 1);
 
             entrypointThread.mutate(formVar, VariableMutationType.ASSIGN, formOutput);
 
@@ -94,7 +94,7 @@ public Workflow buildDeadlineReassignmentWorkflowUserWithoutGroup() {
 
             UserTaskOutput formOutput = entrypointThread.assignUserTask(USER_TASK_DEF_NAME, "test-user-id", null);
 
-            entrypointThread.reassignUserTask(formOutput, null, "test-it-department", 4);
+            entrypointThread.reassignUserTask(formOutput, null, "test-it-department", 1);
 
             entrypointThread.mutate(formVar, VariableMutationType.ASSIGN, formOutput);
 
diff --git a/server/src/test/java/e2e/VarSubErrorTest.java b/server/src/test/java/e2e/VarSubErrorTest.java
index 1ca7a364d..62f2cf568 100644
--- a/server/src/test/java/e2e/VarSubErrorTest.java
+++ b/server/src/test/java/e2e/VarSubErrorTest.java
@@ -1,67 +1,210 @@
 package e2e;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import io.grpc.Status.Code;
+import io.grpc.StatusRuntimeException;
+import io.littlehorse.sdk.common.LHLibUtil;
+import io.littlehorse.sdk.common.proto.Failure;
+import io.littlehorse.sdk.common.proto.LHErrorType;
 import io.littlehorse.sdk.common.proto.LHStatus;
-import io.littlehorse.sdk.common.proto.VariableMutationType;
+import io.littlehorse.sdk.common.proto.LittleHorseGrpc.LittleHorseBlockingStub;
+import io.littlehorse.sdk.common.proto.RunWfRequest;
+import io.littlehorse.sdk.common.proto.TaskAttempt;
+import io.littlehorse.sdk.common.proto.TaskAttempt.ResultCase;
+import io.littlehorse.sdk.common.proto.TaskStatus;
 import io.littlehorse.sdk.common.proto.VariableType;
-import io.littlehorse.sdk.wfsdk.NodeOutput;
-import io.littlehorse.sdk.wfsdk.SpawnedThread;
-import io.littlehorse.sdk.wfsdk.SpawnedThreads;
+import io.littlehorse.sdk.common.proto.VariableValue.ValueCase;
+import io.littlehorse.sdk.common.proto.WfRunId;
+import io.littlehorse.sdk.common.util.Arg;
 import io.littlehorse.sdk.wfsdk.WfRunVariable;
 import io.littlehorse.sdk.wfsdk.Workflow;
+import io.littlehorse.sdk.worker.LHTaskMethod;
 import io.littlehorse.test.LHTest;
 import io.littlehorse.test.LHWorkflow;
 import io.littlehorse.test.WorkflowVerifier;
+import java.util.HashMap;
 import java.util.Map;
-import org.assertj.core.api.Assertions;
+import java.util.UUID;
 import org.junit.jupiter.api.Test;
 
 @LHTest
 public class VarSubErrorTest {
 
-    // Need to put it here so that we can access it in an illegal manner
-    private WfRunVariable variableOnlyAccessibleInChild;
-
+    private LittleHorseBlockingStub client;
     private WorkflowVerifier verifier;
 
-    @LHWorkflow("access-out-of-scope-var")
-    private Workflow accessOutOfScopeVarWf;
+    @LHWorkflow("var-type-validations")
+    private Workflow varTypeValidationsWf;
 
     @Test
-    void shouldThrowVarSubErrorOnIllegalAccess() {
-        verifier.prepareRun(accessOutOfScopeVarWf)
-                // this WfRun should be terminated on the first RPC since there are no
-                // TaskRun's, ExternalEvents, or User Tasks. Everything is synchronous.
-                //
-                // Therefore, we should NOT do a waitForStatus().
+    void shouldThrowInvalidArgumentIfRequiredVarMissing() {
+        // The RunWf RPC would throw an error, and currently the test framework doesn't have
+        // a way to test this. So we first run a simple test with the framework to ensure
+        // that the WfSpec gets created.
+        verifier.prepareRun(
+                        varTypeValidationsWf,
+                        Arg.of("input-int", 2),
+                        Arg.of("input-json", Map.of("int", 1, "string", "hello")))
+                .waitForStatus(LHStatus.COMPLETED)
+                .start();
+
+        String wfRunId = UUID.randomUUID().toString();
+
+        // Now we run our test manually.
+        assertThatThrownBy(() -> {
+                    client.runWf(RunWfRequest.newBuilder()
+                            .setWfSpecName("var-type-validations")
+                            .setId(wfRunId)
+                            .build());
+                })
+                .matches(exn -> {
+                    assertThat(exn).isInstanceOf(StatusRuntimeException.class);
+                    StatusRuntimeException sre = (StatusRuntimeException) exn;
+                    assertThat(sre.getStatus().getCode()).isEqualTo(Code.INVALID_ARGUMENT);
+
+                    return sre.getMessage().toLowerCase().contains("input-int");
+                });
+
+        // Next, we need to make sure the WfRun wasn't actually saved since it had invalid variables.
+        assertThatThrownBy(() -> {
+                    client.getWfRun(WfRunId.newBuilder().setId(wfRunId).build());
+                })
+                .matches(exn -> {
+                    assertThat(exn).isInstanceOf(StatusRuntimeException.class);
+                    StatusRuntimeException sre = (StatusRuntimeException) exn;
+                    return sre.getStatus().getCode() == Code.NOT_FOUND;
+                });
+
+        // Finally, we are going to test that RunWf throws an error when we provide the wrong types.
+        assertThatThrownBy(() -> {
+                    client.runWf(RunWfRequest.newBuilder()
+                            .setWfSpecName("var-type-validations")
+                            .setId(wfRunId)
+                            .putVariables("input-int", LHLibUtil.objToVarVal("not-an-int"))
+                            .build());
+                })
+                .matches(exn -> {
+                    assertThat(exn).isInstanceOf(StatusRuntimeException.class);
+                    StatusRuntimeException sre = (StatusRuntimeException) exn;
+                    assertThat(sre.getStatus().getCode()).isEqualTo(Code.INVALID_ARGUMENT);
+
+                    return sre.getMessage().toLowerCase().contains("input-int");
+                });
+    }
+
+    @Test
+    void shouldFindVarSubErrorOnFirstNodeRun() {
+        verifier.prepareRun(
+                        varTypeValidationsWf,
+                        Arg.of("input-int", 123),
+                        Arg.of("input-json", Map.of("int", "notanint", "str", "yod")))
+                .waitForStatus(LHStatus.ERROR)
+                .thenVerifyNodeRun(0, 1, nr -> {
+                    assertThat(nr.getFailuresCount()).isEqualTo(1);
+                    Failure failure = nr.getFailures(0);
+                    assertThat(failure.getFailureName()).isEqualTo(LHErrorType.VAR_SUB_ERROR.toString());
+                })
+                .start();
+    }
+
+    @Test
+    void shouldThrowVarSubErrorWhenKeyMissingFromJsonObj() {
+        verifier.prepareRun(varTypeValidationsWf, Arg.of("input-int", 123), Arg.of("input-json", Map.of("str", "yoda")))
+                .waitForStatus(LHStatus.ERROR)
+                .thenVerifyNodeRun(0, 1, nodeRun -> {
+                    assertThat(nodeRun.getFailuresCount()).isEqualTo(1);
+                    Failure failure = nodeRun.getFailures(0);
+                    assertThat(failure.getFailureName()).isEqualTo(LHErrorType.VAR_SUB_ERROR.toString());
+                    assertThat(failure.getMessage()).contains("$.int");
+                })
+                .start();
+    }
+
+    @Test
+    void shouldBeAbleToPassNullAsIntIntoTaskRunAndTaskRunShouldFail() {
+        Map jsonStuff = new HashMap<>();
+        jsonStuff.put("str", "my-str");
+        jsonStuff.put("int", null);
+
+        verifier.prepareRun(varTypeValidationsWf, Arg.of("input-int", 123), Arg.of("input-json", jsonStuff))
+                .waitForStatus(LHStatus.ERROR)
+                .thenVerifyTaskRun(0, 1, taskRun -> {
+                    assertThat(taskRun.getStatus()).isEqualTo(TaskStatus.TASK_FAILED);
+
+                    assertThat(taskRun.getInputVariables(0).getValue().getValueCase())
+                            .isEqualTo(ValueCase.VALUE_NOT_SET);
+                    TaskAttempt attempt = taskRun.getAttempts(0);
+                    assertThat(attempt.getResultCase()).isEqualTo(ResultCase.ERROR);
+                })
+                .start();
+    }
+
+    @Test
+    void shouldNotCastStuffToString() {
+        verifier.prepareRun(
+                        varTypeValidationsWf,
+                        Arg.of("input-int", 123),
+                        Arg.of("input-json", Map.of("int", 1234, "string", 5432)))
+                .waitForStatus(LHStatus.ERROR)
+                .thenVerifyWfRun(wfRun -> {
+                    assertThat(wfRun.getThreadRuns(0).getCurrentNodePosition()).isEqualTo(3);
+                })
+                .thenVerifyNodeRun(0, 3, nodeRun -> {
+                    assertThat(nodeRun.getFailuresCount()).isEqualTo(1);
+                    Failure failure = nodeRun.getFailures(0);
+                    assertThat(failure.getFailureName()).isEqualTo(LHErrorType.VAR_SUB_ERROR.toString());
+                    assertThat(failure.getMessage()).contains("STR");
+                })
+                .start();
+    }
+
+    @Test
+    void shouldNotCastDoubleToInt() {
+        verifier.prepareRun(
+                        varTypeValidationsWf,
+                        Arg.of("input-int", 123),
+                        Arg.of("input-json", Map.of("int", 12.3, "string", 5432)))
+                .waitForStatus(LHStatus.ERROR)
                 .thenVerifyWfRun(wfRun -> {
-                    Assertions.assertThat(wfRun.getStatus()).isEqualTo(LHStatus.ERROR);
-                    Assertions.assertThat(wfRun.getThreadRuns(1).getStatus()).isEqualTo(LHStatus.EXCEPTION);
-                    Assertions.assertThat(wfRun.getThreadRuns(2).getStatus()).isEqualTo(LHStatus.ERROR);
+                    assertThat(wfRun.getThreadRuns(0).getCurrentNodePosition()).isEqualTo(1);
+                })
+                .thenVerifyNodeRun(0, 1, nodeRun -> {
+                    assertThat(nodeRun.getFailuresCount()).isEqualTo(1);
+                    Failure failure = nodeRun.getFailures(0);
+                    assertThat(failure.getFailureName()).isEqualTo(LHErrorType.VAR_SUB_ERROR.toString());
+                    assertThat(failure.getMessage()).contains("DOUBLE");
                 })
                 .start();
     }
 
-    @LHWorkflow("access-out-of-scope-var")
-    public Workflow getAccessOutOfScopeVarWf() {
-        return Workflow.newWorkflow("access-out-of-scope-var", wf -> {
-            SpawnedThread spawnedThread = wf.spawnThread(
-                    child -> {
-                        variableOnlyAccessibleInChild = child.addVariable("out-of-scope-var", VariableType.BOOL);
-                        child.fail("some-failure", "some failure message");
-                    },
-                    "child-thread",
-                    Map.of());
-
-            // In order to "trick" the SDK into creating a WfSpec that accesses an
-            // out-of-scope variable, we have to use a failure handler on a waitForThreads
-            // node.
-            NodeOutput nodeThatWillFail = wf.waitForThreads(SpawnedThreads.of(spawnedThread));
-
-            wf.handleException(nodeThatWillFail, handler -> {
-                // We once had an NPE in which accessing a variable from the child in this thread
-                // was permitted by the SDK but the server threw an orzdash.
-                handler.mutate(variableOnlyAccessibleInChild, VariableMutationType.ASSIGN, true);
-            });
+    /*
+     * Allows testers to check that variable type validations are working properly on the following:
+     * - input variables
+     * - input to first task run
+     * - input to later task runs
+     */
+    @LHWorkflow("var-type-validations")
+    public Workflow getVarTypeValidationsWf() {
+        return Workflow.newWorkflow("var-type-validations", wf -> {
+            WfRunVariable inputInt =
+                    wf.addVariable("input-int", VariableType.INT).required();
+            WfRunVariable inputJson = wf.addVariable("input-json", VariableType.JSON_OBJ);
+
+            wf.execute("takes-in-int", inputJson.jsonPath("$.int"));
+            wf.execute("takes-in-int", inputInt);
+            wf.execute("takes-in-string", inputJson.jsonPath("$.string"));
         });
     }
+
+    @LHTaskMethod("takes-in-int")
+    public int obiWan(int input) {
+        return input + 1;
+    }
+
+    @LHTaskMethod("takes-in-string")
+    public String anakin(String input) {
+        return input + "!";
+    }
 }
diff --git a/server/src/test/java/e2e/WaitForThreadsTest.java b/server/src/test/java/e2e/WaitForThreadsTest.java
index 9dd9a6803..ab059c24c 100644
--- a/server/src/test/java/e2e/WaitForThreadsTest.java
+++ b/server/src/test/java/e2e/WaitForThreadsTest.java
@@ -9,7 +9,6 @@
 import io.littlehorse.sdk.common.proto.TaskStatus;
 import io.littlehorse.sdk.common.proto.VariableMutationType;
 import io.littlehorse.sdk.common.proto.VariableType;
-import io.littlehorse.sdk.common.proto.WaitForThreadsPolicy;
 import io.littlehorse.sdk.common.proto.WaitForThreadsRun;
 import io.littlehorse.sdk.wfsdk.NodeOutput;
 import io.littlehorse.sdk.wfsdk.SpawnedThread;
@@ -25,6 +24,7 @@
 import java.util.Map;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
+import lombok.AllArgsConstructor;
 import org.junit.jupiter.api.Test;
 
 @LHTest(
@@ -201,8 +201,7 @@ public Workflow buildWaitForThreadsWithExceptionHandlerWorkflow() {
             SpawnedThread p3Thread =
                     thread.spawnThread(buildChildThread.apply(person3Approved, "person-3"), "person-3", null);
 
-            NodeOutput nodeOutput = thread.waitForThreads(SpawnedThreads.of(p1Thread, p2Thread, p3Thread))
-                    .withPolicy(WaitForThreadsPolicy.STOP_ON_FAILURE);
+            NodeOutput nodeOutput = thread.waitForThreads(SpawnedThreads.of(p1Thread, p2Thread, p3Thread));
 
             thread.handleException(nodeOutput, "denied-by-user", xnHandler -> {
                 xnHandler.execute("exc-handler");
@@ -255,8 +254,7 @@ public Workflow buildWaitForThreadsWithoutExceptionHandlerWorkflow() {
             SpawnedThread p3Thread =
                     thread.spawnThread(buildChildThread.apply(person3Approved, "person-3"), "person-3", null);
 
-            thread.waitForThreads(SpawnedThreads.of(p1Thread, p2Thread, p3Thread))
-                    .withPolicy(WaitForThreadsPolicy.STOP_ON_FAILURE);
+            thread.waitForThreads(SpawnedThreads.of(p1Thread, p2Thread, p3Thread));
 
             // Tell the reminder workflow to stop
             thread.mutate(allApproved, VariableMutationType.ASSIGN, true);
@@ -279,7 +277,7 @@ public Workflow buildSimpleWaitForThreadWorkflow() {
             SpawnedThread child2 = thread.spawnThread(
                     buildSpawnThread.apply("input2", "thread-2-event"), "child-2", Map.of("input2", Map.of()));
 
-            thread.waitForThreads(SpawnedThreads.of(child1, child2)).withPolicy(WaitForThreadsPolicy.STOP_ON_FAILURE);
+            thread.waitForThreads(SpawnedThreads.of(child1, child2));
         });
     }
 
@@ -294,3 +292,9 @@ public int addOne(int input) {
         return input + 1;
     }
 }
+
+@AllArgsConstructor
+class MyInput {
+    public String endResult;
+    public boolean executeTask;
+}
diff --git a/server/src/test/java/e2e/SequentialWorkflowTest.java b/server/src/test/java/e2e/WorkerContextTest.java
similarity index 68%
rename from server/src/test/java/e2e/SequentialWorkflowTest.java
rename to server/src/test/java/e2e/WorkerContextTest.java
index 1476f2ed0..cdef328f2 100644
--- a/server/src/test/java/e2e/SequentialWorkflowTest.java
+++ b/server/src/test/java/e2e/WorkerContextTest.java
@@ -1,5 +1,7 @@
 package e2e;
 
+import static org.assertj.core.api.Assertions.assertThat;
+
 import io.littlehorse.sdk.common.proto.LHStatus;
 import io.littlehorse.sdk.common.proto.TaskStatus;
 import io.littlehorse.sdk.wfsdk.Workflow;
@@ -9,15 +11,14 @@
 import io.littlehorse.test.LHTest;
 import io.littlehorse.test.LHWorkflow;
 import io.littlehorse.test.WorkflowVerifier;
-import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 @LHTest
-public class SequentialWorkflowTest {
+public class WorkerContextTest {
 
     private WorkflowVerifier workflowVerifier;
 
-    @LHWorkflow("simple-sequential-wf")
+    @LHWorkflow("worker-context-test")
     private Workflow workflow;
 
     @Test
@@ -27,11 +28,12 @@ public void simpleSequentialWorkflowExecution() {
                 .waitForStatus(LHStatus.COMPLETED)
                 .waitForTaskStatus(0, 1, TaskStatus.TASK_SUCCESS)
                 .waitForTaskStatus(0, 2, TaskStatus.TASK_SUCCESS)
-                .thenVerifyTaskRunResult(
-                        0,
-                        1,
-                        variableValue ->
-                                Assertions.assertTrue(variableValue.getStr().contains("hello there from wfRun")))
+                .thenVerifyAllTaskRuns(0, taskRuns -> {
+                    assertThat(taskRuns.get(0).getAttempts(0).getOutput().getStr())
+                            .contains(" on nodeRun 1");
+                    assertThat(taskRuns.get(1).getAttempts(0).getOutput().getStr())
+                            .contains(" on nodeRun 2");
+                })
                 .start();
     }
 
@@ -49,15 +51,15 @@ public void readYourOwnWritesTest() {
         }
     }
 
-    @LHWorkflow("simple-sequential-wf")
+    @LHWorkflow("worker-context-test")
     public Workflow buildWorkflow() {
-        return new WorkflowImpl("simple-sequential-wf", thread -> {
-            thread.execute("aa-simple");
-            thread.execute("aa-simple");
+        return new WorkflowImpl("worker-context-test", thread -> {
+            thread.execute("worker-context-test");
+            thread.execute("worker-context-test");
         });
     }
 
-    @LHTaskMethod("aa-simple")
+    @LHTaskMethod("worker-context-test")
     public String obiWan(WorkerContext context) {
 
         return ("hello there from wfRun "
diff --git a/test-utils/src/main/java/io/littlehorse/test/LHExtension.java b/test-utils/src/main/java/io/littlehorse/test/LHExtension.java
index 16165cb12..7836f63d5 100644
--- a/test-utils/src/main/java/io/littlehorse/test/LHExtension.java
+++ b/test-utils/src/main/java/io/littlehorse/test/LHExtension.java
@@ -83,6 +83,9 @@ public void postProcessTestInstance(Object testInstance, ExtensionContext contex
 
     private void maybeCreateTenantAndPrincipal(TestContext testContext) {
         String principalId = getPrincipalId();
+        if (testContext.getConfig().getTenantId() == null) {
+            return;
+        }
         try {
             testContext
                     .getAnonymousClient()
diff --git a/test-utils/src/main/java/io/littlehorse/test/internal/ExternalTestBootstrapper.java b/test-utils/src/main/java/io/littlehorse/test/internal/ExternalTestBootstrapper.java
index 1083f7897..037bb0180 100644
--- a/test-utils/src/main/java/io/littlehorse/test/internal/ExternalTestBootstrapper.java
+++ b/test-utils/src/main/java/io/littlehorse/test/internal/ExternalTestBootstrapper.java
@@ -1,5 +1,7 @@
 package io.littlehorse.test.internal;
 
+import io.littlehorse.common.model.getable.objectId.PrincipalIdModel;
+import io.littlehorse.common.model.getable.objectId.TenantIdModel;
 import io.littlehorse.sdk.common.config.LHConfig;
 import io.littlehorse.sdk.common.proto.LittleHorseGrpc.LittleHorseBlockingStub;
 import java.nio.file.Files;
@@ -11,13 +13,22 @@ public class ExternalTestBootstrapper implements TestBootstrapper {
     private final LHConfig workerConfig;
     private final LittleHorseBlockingStub lhClient;
     private Path configPath = Path.of(System.getProperty("user.home"), LH_CONFIG_FILE);
+    private LittleHorseBlockingStub anonymousClient;
 
-    public ExternalTestBootstrapper() {
+    public ExternalTestBootstrapper(PrincipalIdModel principalId) {
         if (Files.notExists(configPath)) {
             throw new IllegalStateException(String.format("Configuration file %s doesn't exist", LH_CONFIG_FILE));
         }
         workerConfig = new LHConfig(configPath.toString());
-        lhClient = workerConfig.getBlockingStub();
+        anonymousClient = workerConfig.getBlockingStub();
+        if (principalId != null && workerConfig.getTenantId() != null) {
+            lhClient = workerConfig
+                    .getBlockingStub()
+                    .withCallCredentials(new StandaloneTestBootstrapper.MockCallCredentials(
+                            principalId, new TenantIdModel(workerConfig.getTenantId())));
+        } else {
+            lhClient = anonymousClient;
+        }
     }
 
     @Override
diff --git a/test-utils/src/main/java/io/littlehorse/test/internal/StandaloneTestBootstrapper.java b/test-utils/src/main/java/io/littlehorse/test/internal/StandaloneTestBootstrapper.java
index 5966aea79..62f50c91d 100644
--- a/test-utils/src/main/java/io/littlehorse/test/internal/StandaloneTestBootstrapper.java
+++ b/test-utils/src/main/java/io/littlehorse/test/internal/StandaloneTestBootstrapper.java
@@ -1,9 +1,6 @@
 package io.littlehorse.test.internal;
 
 import com.google.protobuf.Empty;
-import io.grpc.CallCredentials;
-import io.grpc.Metadata;
-import io.grpc.Status;
 import io.grpc.StatusRuntimeException;
 import io.littlehorse.common.LHServerConfig;
 import io.littlehorse.common.model.getable.objectId.PrincipalIdModel;
@@ -11,13 +8,11 @@
 import io.littlehorse.sdk.common.config.LHConfig;
 import io.littlehorse.sdk.common.proto.LittleHorseGrpc.LittleHorseBlockingStub;
 import io.littlehorse.server.KafkaStreamsServerImpl;
-import io.littlehorse.server.auth.ServerAuthorizer;
 import io.littlehorse.test.exception.LHTestInitializationException;
 import java.io.IOException;
 import java.time.Duration;
 import java.util.Properties;
 import java.util.UUID;
-import java.util.concurrent.Executor;
 import java.util.concurrent.TimeUnit;
 import org.apache.kafka.clients.admin.NewTopic;
 import org.awaitility.Awaitility;
@@ -119,29 +114,4 @@ private Properties testClientProperties() {
         configs.put(LHConfig.TENANT_ID_KEY, tenantId);
         return configs;
     }
-
-    private static final class MockCallCredentials extends CallCredentials {
-
-        private final PrincipalIdModel principalId;
-        private final TenantIdModel tenantId;
-
-        MockCallCredentials(final PrincipalIdModel principalId, final TenantIdModel tenantId) {
-            this.principalId = principalId;
-            this.tenantId = tenantId;
-        }
-
-        @Override
-        public void applyRequestMetadata(RequestInfo requestInfo, Executor executor, MetadataApplier metadataApplier) {
-            executor.execute(() -> {
-                try {
-                    Metadata headers = new Metadata();
-                    headers.put(ServerAuthorizer.CLIENT_ID, principalId.getId());
-                    headers.put(ServerAuthorizer.TENANT_ID, tenantId.getId());
-                    metadataApplier.apply(headers);
-                } catch (Exception e) {
-                    metadataApplier.fail(Status.UNAUTHENTICATED.withCause(e));
-                }
-            });
-        }
-    }
 }
diff --git a/test-utils/src/main/java/io/littlehorse/test/internal/TestBootstrapper.java b/test-utils/src/main/java/io/littlehorse/test/internal/TestBootstrapper.java
index f4ab7dedb..f7c22ddb0 100644
--- a/test-utils/src/main/java/io/littlehorse/test/internal/TestBootstrapper.java
+++ b/test-utils/src/main/java/io/littlehorse/test/internal/TestBootstrapper.java
@@ -1,7 +1,14 @@
 package io.littlehorse.test.internal;
 
+import io.grpc.CallCredentials;
+import io.grpc.Metadata;
+import io.grpc.Status;
+import io.littlehorse.common.model.getable.objectId.PrincipalIdModel;
+import io.littlehorse.common.model.getable.objectId.TenantIdModel;
 import io.littlehorse.sdk.common.config.LHConfig;
 import io.littlehorse.sdk.common.proto.LittleHorseGrpc.LittleHorseBlockingStub;
+import io.littlehorse.server.auth.ServerAuthorizer;
+import java.util.concurrent.Executor;
 
 public interface TestBootstrapper {
     LHConfig getWorkerConfig();
@@ -9,4 +16,29 @@ public interface TestBootstrapper {
     LittleHorseBlockingStub getLhClient();
 
     LittleHorseBlockingStub getAnonymousClient();
+
+    final class MockCallCredentials extends CallCredentials {
+
+        private final PrincipalIdModel principalId;
+        private final TenantIdModel tenantId;
+
+        MockCallCredentials(final PrincipalIdModel principalId, final TenantIdModel tenantId) {
+            this.principalId = principalId;
+            this.tenantId = tenantId;
+        }
+
+        @Override
+        public void applyRequestMetadata(RequestInfo requestInfo, Executor executor, MetadataApplier metadataApplier) {
+            executor.execute(() -> {
+                try {
+                    Metadata headers = new Metadata();
+                    headers.put(ServerAuthorizer.CLIENT_ID, principalId.getId());
+                    headers.put(ServerAuthorizer.TENANT_ID, tenantId.getId());
+                    metadataApplier.apply(headers);
+                } catch (Exception e) {
+                    metadataApplier.fail(Status.UNAUTHENTICATED.withCause(e));
+                }
+            });
+        }
+    }
 }