diff --git a/apps/sim/executor/variables/resolver.test.ts b/apps/sim/executor/variables/resolver.test.ts index c517dc83d7c..226d74b356e 100644 --- a/apps/sim/executor/variables/resolver.test.ts +++ b/apps/sim/executor/variables/resolver.test.ts @@ -286,6 +286,47 @@ async function evaluateResolvedCondition( } } +function createOffloadEnv(language: string, producerOutput: Record) { + const { block, ctx } = createResolver(language) + const producer = createBlock('producer', 'Producer', BlockType.API) + const state = new ExecutionState() + state.setBlockOutput('producer', producerOutput) + const workflow: SerializedWorkflow = { + version: '1', + blocks: [producer, block], + connections: [], + loops: {}, + parallels: {}, + } + const resolver = new VariableResolver(workflow, {}, state) + const durableCtx = { + ...ctx, + blockStates: state.getBlockStates(), + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + largeValueKeys: [] as string[], + } as ExecutionContext + return { block, resolver, durableCtx } +} + +function largeRef(size: number) { + return { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'string', + size, + key: 'execution/workspace-1/workflow-1/execution-1/large-value-lv_ABCDEFGHIJKL.json', + executionId: 'execution-1', + } +} + +function resolveAgainstProducer(language: string, output: Record, code: string) { + const { block, resolver, durableCtx } = createOffloadEnv(language, output) + return resolver.resolveInputsForFunctionBlock(durableCtx, 'function', { code }, block) +} + describe('VariableResolver function block inputs', () => { it('inlines only structurally inert condition literals and defers the rest to the compiler', async () => { const { ctx, resolver } = createResolver() @@ -924,116 +965,103 @@ describe('VariableResolver function block inputs', () => { }) it('fails whole large value refs for Function runtimes without lazy helpers', async () => { - const { block, ctx } = createResolver('python') - const state = new ExecutionState() - state.setBlockOutput('producer', { - result: { - __simLargeValueRef: true, - version: 1, - id: 'lv_ABCDEFGHIJKL', - kind: 'object', - size: 12 * 1024 * 1024, - key: 'execution/workspace-1/workflow-1/execution-1/large-value-lv_ABCDEFGHIJKL.json', - executionId: 'execution-1', - }, - }) - const workflow: SerializedWorkflow = { - version: '1', - blocks: [createBlock('producer', 'Producer', BlockType.API), block], - connections: [], - loops: {}, - parallels: {}, - } - const largeResolver = new VariableResolver(workflow, {}, state) - const largeCtx = { - ...ctx, - blockStates: state.getBlockStates(), - } as ExecutionContext - await expect( - largeResolver.resolveInputsForFunctionBlock( - largeCtx, - 'function', - { code: 'return ' }, - block + resolveAgainstProducer( + 'python', + { result: largeRef(12 * 1024 * 1024) }, + 'return ' ) - ).rejects.toThrow('This execution value is too large to inline') + ).rejects.toThrow( + ' is 12.0 MB, too large to pass into this block. Select a smaller field, or read it in JavaScript without imports.' + ) }) it('fails whole large value refs for JavaScript with imports', async () => { - const { block, ctx } = createResolver('javascript') - const state = new ExecutionState() - state.setBlockOutput('producer', { - result: { - __simLargeValueRef: true, - version: 1, - id: 'lv_ABCDEFGHIJKL', - kind: 'object', - size: 12 * 1024 * 1024, - key: 'execution/workspace-1/workflow-1/execution-1/large-value-lv_ABCDEFGHIJKL.json', - executionId: 'execution-1', - }, - }) - const workflow: SerializedWorkflow = { - version: '1', - blocks: [createBlock('producer', 'Producer', BlockType.API), block], - connections: [], - loops: {}, - parallels: {}, - } - const largeResolver = new VariableResolver(workflow, {}, state) - const largeCtx = { - ...ctx, - blockStates: state.getBlockStates(), - } as ExecutionContext - await expect( - largeResolver.resolveInputsForFunctionBlock( - largeCtx, - 'function', - { code: "import x from 'x'\nreturn " }, - block + resolveAgainstProducer( + 'javascript', + { result: largeRef(12 * 1024 * 1024) }, + "import x from 'x'\nreturn " ) - ).rejects.toThrow('This execution value is too large to inline') + ).rejects.toThrow( + ' is 12.0 MB, too large to pass into this block. Select a smaller field, or read it in JavaScript without imports.' + ) }) - it('fails nested large value refs for JavaScript instead of leaking ref markers', async () => { - const { block, ctx } = createResolver('javascript') - const state = new ExecutionState() - state.setBlockOutput('producer', { - result: { - rows: { - __simLargeValueRef: true, - version: 1, - id: 'lv_ABCDEFGHIJKL', - kind: 'array', - size: 12 * 1024 * 1024, - key: 'execution/workspace-1/workflow-1/execution-1/large-value-lv_ABCDEFGHIJKL.json', - executionId: 'execution-1', - }, - }, + describe('nested large values', () => { + it('names the nested field instead of leaking ref markers', async () => { + await expect( + resolveAgainstProducer( + 'javascript', + { result: { rows: largeRef(12 * 1024 * 1024) } }, + 'return .rows.length' + ) + ).rejects.toThrow( + ' contains a 12.0 MB value at . Reference that field directly.' + ) }) - const workflow: SerializedWorkflow = { - version: '1', - blocks: [createBlock('producer', 'Producer', BlockType.API), block], - connections: [], - loops: {}, - parallels: {}, - } - const largeResolver = new VariableResolver(workflow, {}, state) - const largeCtx = { - ...ctx, - blockStates: state.getBlockStates(), - } as ExecutionContext - await expect( - largeResolver.resolveInputsForFunctionBlock( - largeCtx, - 'function', - { code: 'return .rows.length' }, - block + it('names the array index of a large value nested in an array', async () => { + await expect( + resolveAgainstProducer( + 'javascript', + { contents: [largeRef(10.5 * 1024 * 1024)] }, + 'return [0].slice(0, 10)' + ) + ).rejects.toThrow( + ' contains a 10.5 MB value at . Reference that field directly.' ) - ).rejects.toThrow('This execution value contains nested large values') + }) + + it('names a nested array manifest without descending into its chunks', async () => { + const chunk = largeRef(9 * 1024 * 1024) + const manifest = { + __simLargeArrayManifest: true, + version: 2, + kind: 'array', + totalCount: 2, + chunkCount: 1, + byteSize: chunk.size, + chunks: [{ ref: chunk, count: 2, byteSize: chunk.size }], + preview: [], + } + await expect( + resolveAgainstProducer( + 'javascript', + { result: { items: manifest } }, + 'return .items.length' + ) + ).rejects.toThrow( + ' contains a 9.0 MB value at . Reference that field directly.' + ) + }) + + it('does not suggest a direct reference where it could not be loaded either', async () => { + await expect( + resolveAgainstProducer( + 'python', + { contents: [largeRef(10.5 * 1024 * 1024)] }, + 'return ' + ) + ).rejects.toThrow( + ' contains a 10.5 MB value at . Select a smaller field, or read it in JavaScript without imports.' + ) + }) + + it.each(['report.txt', '0'])( + 'omits a field reference that the key %s cannot be written as', + async (key) => { + await expect( + resolveAgainstProducer( + 'javascript', + { result: { [key]: largeRef(10.5 * 1024 * 1024) } }, + 'return ' + ) + ).rejects.toThrow( + ' contains a 10.5 MB value. Reference the field you need directly.' + ) + } + ) }) it('breaks JavaScript string literals around quoted block references', async () => { @@ -1148,30 +1176,6 @@ describe('VariableResolver function block inputs', () => { describe('VariableResolver function context overflow offload', () => { const REF_KEY = 'execution/workspace-1/workflow-1/execution-1/large-value-lv_ABCDEFGHIJKL.json' - function createOffloadEnv(language: string, producerOutput: Record) { - const { block, ctx } = createResolver(language) - const producer = createBlock('producer', 'Producer', BlockType.API) - const state = new ExecutionState() - state.setBlockOutput('producer', producerOutput) - const workflow: SerializedWorkflow = { - version: '1', - blocks: [producer, block], - connections: [], - loops: {}, - parallels: {}, - } - const resolver = new VariableResolver(workflow, {}, state) - const durableCtx = { - ...ctx, - blockStates: state.getBlockStates(), - workspaceId: 'workspace-1', - workflowId: 'workflow-1', - executionId: 'execution-1', - largeValueKeys: [] as string[], - } as ExecutionContext - return { block, resolver, durableCtx } - } - beforeEach(() => { mockStoreLargeValue.mockReset() mockStoreLargeValue.mockResolvedValue({ diff --git a/apps/sim/executor/variables/resolver.ts b/apps/sim/executor/variables/resolver.ts index 7184e4660db..a1249f793ed 100644 --- a/apps/sim/executor/variables/resolver.ts +++ b/apps/sim/executor/variables/resolver.ts @@ -6,7 +6,6 @@ import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manif import { containsLargeValueRef, formatLargeValueSize, - getLargeValueMaterializationError, isLargeValueRef, type LargeValueRef, } from '@/lib/execution/payloads/large-value-ref' @@ -87,9 +86,81 @@ function measureJson(value: unknown): { json: string; size: number } | null { } } -function getNestedLargeValueMaterializationError(): Error { +interface LocatedLargeValue { + path: Array + size: number +} + +/** + * An object key a reference can reach with `.key`: no whitespace, path, or operator characters, + * and not digits alone, which path navigation reads as an array index. + */ +const REFERENCEABLE_KEY = /^(?!\d+$)[^\s.[\]+*/=<>!&|]+$/ + +/** How to use a large value in a runtime that cannot load one on demand. */ +const LARGE_VALUE_RUNTIME_HINT = 'Select a smaller field, or read it in JavaScript without imports.' + +/** + * Finds the first offloaded value in a resolved value, with its path and stored size. + * + * Stops at the first hit, like `containsLargeValueRef`, and never descends into an array + * manifest's chunks — the manifest itself is the field an author can reference. + */ +function findLargeValue(value: unknown): LocatedLargeValue | null { + const path: Array = [] + const seen = new WeakSet() + const visit = (current: unknown): LocatedLargeValue | null => { + if (!current || typeof current !== 'object') return null + if (isLargeValueRef(current)) return { path: [...path], size: current.size } + if (isLargeArrayManifest(current)) return { path: [...path], size: current.byteSize } + if (seen.has(current)) return null + seen.add(current) + const keys = Array.isArray(current) ? current.keys() : Object.keys(current) + for (const key of keys) { + path.push(key) + const found = visit((current as Record)[key]) + path.pop() + if (found) return found + } + return null + } + return visit(value) +} + +/** Extends `` with a nested path, or `null` when a key cannot be written as one. */ +function appendReferencePath(reference: string, path: Array): string | null { + let suffix = '' + for (const segment of path) { + if (typeof segment === 'number') suffix += `[${segment}]` + else if (REFERENCEABLE_KEY.test(segment)) suffix += `${REFERENCE.PATH_DELIMITER}${segment}` + else return null + } + return `${reference.slice(0, -REFERENCE.END.length)}${suffix}${REFERENCE.END}` +} + +/** A Function block reference points straight at a large value its runtime cannot load. */ +function getDirectLargeValueError(reference: string, size: number): Error { return new Error( - 'This execution value contains nested large values. Reference the nested field directly so it can be lazy-loaded.' + `${reference} is ${formatLargeValueSize(size)}, too large to pass into this block. ${LARGE_VALUE_RUNTIME_HINT}` + ) +} + +/** + * A Function block reference holds a large value below its root. Only a reference pointing + * straight at a large value can load it on demand, so this names that field. + */ +function getNestedLargeValueError( + reference: string, + value: unknown, + canLoadOnDemand: boolean +): Error { + const found = findLargeValue(value) + const field = found && appendReferencePath(reference, found.path) + const size = found ? `${formatLargeValueSize(found.size)} ` : '' + const described = `${reference} contains a ${size}value${field ? ` at ${field}` : ''}.` + if (!canLoadOnDemand) return new Error(`${described} ${LARGE_VALUE_RUNTIME_HINT}`) + return new Error( + `${described} Reference ${field ? 'that field' : 'the field you need'} directly.` ) } @@ -575,7 +646,7 @@ export class VariableResolver { index ) if (!lazyReplacement) { - throw getLargeValueMaterializationError(effectiveValue) + throw getDirectLargeValueError(match, effectiveValue.size) } replacement = lazyReplacement } else if (isLargeArrayManifest(effectiveValue)) { @@ -586,11 +657,15 @@ export class VariableResolver { index ) if (!lazyReplacement) { - throw getNestedLargeValueMaterializationError() + throw getDirectLargeValueError(match, effectiveValue.byteSize) } replacement = lazyReplacement } else if (containsLargeValueRef(effectiveValue)) { - throw getNestedLargeValueMaterializationError() + throw getNestedLargeValueError( + match, + effectiveValue, + this.canUseJavaScriptRuntimeHelpers(language, template) + ) } else { const offloadedRef = await this.maybeOffloadInlineFunctionContextValue( ctx, @@ -659,7 +734,7 @@ export class VariableResolver { ) return lazyReplacement } - throw getLargeValueMaterializationError(effectiveValue) + throw getDirectLargeValueError(match, effectiveValue.size) } if (isLargeArrayManifest(effectiveValue)) { @@ -680,11 +755,15 @@ export class VariableResolver { ) return lazyReplacement } - throw getNestedLargeValueMaterializationError() + throw getDirectLargeValueError(match, effectiveValue.byteSize) } if (containsLargeValueRef(effectiveValue)) { - throw getNestedLargeValueMaterializationError() + throw getNestedLargeValueError( + match, + effectiveValue, + this.canUseJavaScriptRuntimeHelpers(language, template) + ) } if (this.canInlineResolvedCodeLiteral(effectiveValue, match)) {