diff --git a/apps/docs/components/ui/image.tsx b/apps/docs/components/ui/image.tsx index 311f1d649e8..f667508a99a 100644 --- a/apps/docs/components/ui/image.tsx +++ b/apps/docs/components/ui/image.tsx @@ -25,7 +25,7 @@ export function Image({ {image} diff --git a/apps/docs/components/ui/video.tsx b/apps/docs/components/ui/video.tsx index a4eb58dd3b2..b4aee094758 100644 --- a/apps/docs/components/ui/video.tsx +++ b/apps/docs/components/ui/video.tsx @@ -45,7 +45,10 @@ export function Video({ playsInline={playsInline} width={width} height={height} - className={cn(className, enableLightbox && 'transition-opacity group-hover:opacity-[0.97]')} + className={cn( + className, + enableLightbox && 'cursor-pointer transition-opacity group-hover:opacity-[0.97]' + )} src={getAssetUrl(src)} /> ) @@ -57,7 +60,7 @@ export function Video({ type='button' onClick={openLightbox} aria-label={`Open ${src} in media viewer`} - className='group block w-full cursor-pointer rounded-xl p-0 text-left' + className='group contents' > {video} diff --git a/apps/docs/content/docs/de/api-reference/getting-started.mdx b/apps/docs/content/docs/de/api-reference/getting-started.mdx index fa9fad0baa3..25c8cfdbf2e 100644 --- a/apps/docs/content/docs/de/api-reference/getting-started.mdx +++ b/apps/docs/content/docs/de/api-reference/getting-started.mdx @@ -109,20 +109,22 @@ curl -X POST https://www.sim.ai/api/workflows/{workflowId}/execute \ -d '{"inputs": {}, "async": true}' ``` -This returns immediately with a `taskId`: +This returns immediately with a `jobId` and `statusUrl`: ```json { "success": true, - "taskId": "job_abc123", - "status": "queued" + "jobId": "job_abc123", + "statusUrl": "https://www.sim.ai/api/jobs/job_abc123", + "message": "Workflow execution started", + "async": true } ``` Poll the [Get Job Status](/api-reference/workflows/getJobStatus) endpoint until the status is `completed` or `failed`: ```bash -curl https://www.sim.ai/api/jobs/{taskId} \ +curl https://www.sim.ai/api/jobs/{jobId} \ -H "X-API-Key: YOUR_API_KEY" ``` diff --git a/apps/docs/content/docs/en/api-reference/getting-started.mdx b/apps/docs/content/docs/en/api-reference/getting-started.mdx index dced7aca61c..038998853cf 100644 --- a/apps/docs/content/docs/en/api-reference/getting-started.mdx +++ b/apps/docs/content/docs/en/api-reference/getting-started.mdx @@ -109,20 +109,22 @@ curl -X POST https://www.sim.ai/api/workflows/{workflowId}/execute \ -d '{"inputs": {}, "async": true}' ``` -This returns immediately with a `taskId`: +This returns immediately with a `jobId` and `statusUrl`: ```json { "success": true, - "taskId": "job_abc123", - "status": "queued" + "jobId": "job_abc123", + "statusUrl": "https://www.sim.ai/api/jobs/job_abc123", + "message": "Workflow execution started", + "async": true } ``` Poll the [Get Job Status](/api-reference/workflows/getJobStatus) endpoint until the status is `completed` or `failed`: ```bash -curl https://www.sim.ai/api/jobs/{taskId} \ +curl https://www.sim.ai/api/jobs/{jobId} \ -H "X-API-Key: YOUR_API_KEY" ``` diff --git a/apps/docs/content/docs/en/api-reference/python.mdx b/apps/docs/content/docs/en/api-reference/python.mdx index 903bac51f1c..d70bb50e3aa 100644 --- a/apps/docs/content/docs/en/api-reference/python.mdx +++ b/apps/docs/content/docs/en/api-reference/python.mdx @@ -80,7 +80,7 @@ result = client.execute_workflow( **Returns:** `WorkflowExecutionResult | AsyncExecutionResult` -When `async_execution=True`, returns immediately with a task ID for polling. Otherwise, waits for completion. +When `async_execution=True`, returns immediately with a `job_id` and `status_url` for polling. Otherwise, waits for completion. ##### get_workflow_status() @@ -117,20 +117,20 @@ if is_ready: Get the status of an async job execution. ```python -status = client.get_job_status("task-id-from-async-execution") +status = client.get_job_status("job-id-from-async-execution") print("Status:", status["status"]) # 'queued', 'processing', 'completed', 'failed' if status["status"] == "completed": print("Output:", status["output"]) ``` **Parameters:** -- `task_id` (str): The task ID returned from async execution +- `task_id` (str): The job ID returned from async execution **Returns:** `Dict[str, Any]` **Response fields:** - `success` (bool): Whether the request was successful -- `taskId` (str): The task ID +- `taskId` (str): The job ID - `status` (str): One of `'queued'`, `'processing'`, `'completed'`, `'failed'`, `'cancelled'` - `metadata` (dict): Contains `startedAt`, `completedAt`, and `duration` - `output` (any, optional): The workflow output (when completed) @@ -270,10 +270,11 @@ class WorkflowExecutionResult: @dataclass class AsyncExecutionResult: success: bool - task_id: str - status: str # 'queued' - created_at: str - links: Dict[str, str] # e.g., {"status": "/api/jobs/{taskId}"} + job_id: str + status_url: str + execution_id: Optional[str] = None + message: str = "" + async_execution: bool = True ``` ### WorkflowStatus @@ -493,17 +494,17 @@ def execute_async(): ) # Check if result is an async execution - if hasattr(result, 'task_id'): - print(f"Task ID: {result.task_id}") - print(f"Status endpoint: {result.links['status']}") + if hasattr(result, 'job_id'): + print(f"Job ID: {result.job_id}") + print(f"Status endpoint: {result.status_url}") # Poll for completion - status = client.get_job_status(result.task_id) + status = client.get_job_status(result.job_id) while status["status"] in ["queued", "processing"]: print(f"Current status: {status['status']}") time.sleep(2) # Wait 2 seconds - status = client.get_job_status(result.task_id) + status = client.get_job_status(result.job_id) if status["status"] == "completed": print("Workflow completed!") @@ -764,7 +765,7 @@ import { FAQ } from '@/components/ui/faq' ` -When `async: true`, returns immediately with a task ID for polling. Otherwise, waits for completion. +When `async: true`, returns immediately with a `jobId` and `statusUrl` for polling. Otherwise, waits for completion. ##### getWorkflowStatus() @@ -131,7 +131,7 @@ if (isReady) { Get the status of an async job execution. ```typescript -const status = await client.getJobStatus('task-id-from-async-execution'); +const status = await client.getJobStatus('job-id-from-async-execution'); console.log('Status:', status.status); // 'queued', 'processing', 'completed', 'failed' if (status.status === 'completed') { console.log('Output:', status.output); @@ -139,13 +139,13 @@ if (status.status === 'completed') { ``` **Parameters:** -- `taskId` (string): The task ID returned from async execution +- `jobId` (string): The job ID returned from async execution **Returns:** `Promise` **Response fields:** - `success` (boolean): Whether the request was successful -- `taskId` (string): The task ID +- `taskId` (string): The job ID - `status` (string): One of `'queued'`, `'processing'`, `'completed'`, `'failed'`, `'cancelled'` - `metadata` (object): Contains `startedAt`, `completedAt`, and `duration` - `output` (any, optional): The workflow output (when completed) @@ -278,12 +278,11 @@ interface WorkflowExecutionResult { ```typescript interface AsyncExecutionResult { success: boolean; - taskId: string; - status: 'queued'; - createdAt: string; - links: { - status: string; // e.g., "/api/jobs/{taskId}" - }; + jobId: string; + statusUrl: string; + executionId?: string; + message: string; + async: true; } ``` @@ -767,17 +766,17 @@ async function executeAsync() { }); // Check if result is an async execution - if ('taskId' in result) { - console.log('Task ID:', result.taskId); - console.log('Status endpoint:', result.links.status); + if ('jobId' in result) { + console.log('Job ID:', result.jobId); + console.log('Status endpoint:', result.statusUrl); // Poll for completion - let status = await client.getJobStatus(result.taskId); + let status = await client.getJobStatus(result.jobId); while (status.status === 'queued' || status.status === 'processing') { console.log('Current status:', status.status); await new Promise(resolve => setTimeout(resolve, 2000)); // Wait 2 seconds - status = await client.getJobStatus(result.taskId); + status = await client.getJobStatus(result.jobId); } if (status.status === 'completed') { @@ -1022,7 +1021,7 @@ import { FAQ } from '@/components/ui/faq' `. Larger values are stored in execution storage and passed around as small references until code explicitly reads them. + +File outputs are metadata-first by default. Referencing ``, ``, or similar metadata does not hydrate file contents. In JavaScript functions without imports, a direct base64 reference like `` is automatically rewritten to a lazy server-side read so the base64 string does not cross the Function request body. + +You can also call the helper explicitly: + +```javascript +const file = ; +const base64 = await sim.files.readBase64(file); +``` + +`sim.files.readBase64(file)`, `sim.files.readText(file)`, `sim.files.readBase64Chunk(file, { offset, length })`, and `sim.files.readTextChunk(file, { offset, length })` read from server-side execution storage under memory caps. `sim.values.read(ref)` can explicitly read a large execution value reference. These helpers are available only in JavaScript functions without imports. JavaScript with imports, Python, and shell do not support these lazy helpers yet. + +Very large full reads can still fail by design; use chunk helpers or return a file when you need to handle more data. + +Use text chunks for text-like files such as logs, CSV, JSONL, and markdown: + +```javascript +const file = ; +const firstMegabyte = await sim.files.readTextChunk(file, { + offset: 0, + length: 1024 * 1024, +}); + +return firstMegabyte.split('\n').slice(0, 10); +``` + +Use base64 chunks for binary files such as images, PDFs, audio, archives, or APIs that expect base64 input: + +```javascript +const file = ; +const firstMegabyteBase64 = await sim.files.readBase64Chunk(file, { + offset: 0, + length: 1024 * 1024, +}); + +return { name: file.name, chunk: firstMegabyteBase64 }; +``` + +Chunk `offset` and `length` are byte-based. For Unicode text, a chunk can split a multi-byte character at the boundary; use text chunks for approximate text processing and prefer smaller structured references when exact parsing matters. + +Avoid passing a full large object into a Function block when you only need one field. For example, prefer `` over `` when the API response is large. If a JavaScript Function without imports references a large execution value, Sim automatically reads it through `sim.values.read(...)` at runtime under memory caps. + +For large generated data, write the result to a file or table with `outputPath`, `outputSandboxPath`, or `outputTable` instead of returning the entire payload inline. + - **Keep functions focused**: Write functions that do one thing well to improve maintainability and debugging - **Handle errors gracefully**: Use try/catch blocks to handle potential errors and provide meaningful error messages - **Test edge cases**: Ensure your code handles unusual inputs, null values, and boundary conditions correctly diff --git a/apps/docs/content/docs/en/blocks/parallel.mdx b/apps/docs/content/docs/en/blocks/parallel.mdx index f3207d901bc..24fccc7ebf3 100644 --- a/apps/docs/content/docs/en/blocks/parallel.mdx +++ b/apps/docs/content/docs/en/blocks/parallel.mdx @@ -34,6 +34,7 @@ Choose between two types of parallel execution: Use this when you need to run the same operation multiple times concurrently. + If the total count is larger than the batch size, Sim runs the work in serial batches while preserving the original result order. ``` Example: Run 5 parallel instances @@ -57,7 +58,7 @@ Choose between two types of parallel execution: /> - Each instance processes one item from the collection simultaneously. + Each instance processes one item from the collection. Large collections run in serial batches while preserving each item's original index. ``` Example: Process ["task1", "task2", "task3"] in parallel @@ -140,6 +141,12 @@ const allResults = ; // Returns: [result1, result2, result3, ...] ``` +For large result sets, reference only the entry or field you need, such as ``. Sim keeps aggregate results indexable by storing oversized entries in execution storage and hydrating them only when an indexed server-side path is explicitly referenced. + +### Batch Size + +Parallel blocks run up to 20 branches at a time by default. Increase the total count or collection size to process more work; Sim will execute the next batch after the current batch finishes. You can lower the batch size to reduce concurrency for rate-limited APIs. + ### Instance Isolation Each parallel instance runs independently: @@ -157,7 +164,7 @@ Each parallel instance runs independently: While parallel execution is faster, be mindful of: - API rate limits when making concurrent requests - Memory usage with large datasets - - Maximum of 20 concurrent instances to prevent resource exhaustion + - Maximum of 20 concurrent instances per batch to prevent resource exhaustion ## Parallel vs Loop @@ -186,6 +193,9 @@ Understanding when to use each:
  • Collection: Array or object to distribute (collection-based)
    • +
  • + Batch size: Number of branches to run concurrently, from 1 to 20 +
    • diff --git a/apps/docs/content/docs/en/execution/api-deployment.mdx b/apps/docs/content/docs/en/execution/api-deployment.mdx index b74a886271c..b7f1de3fbf9 100644 --- a/apps/docs/content/docs/en/execution/api-deployment.mdx +++ b/apps/docs/content/docs/en/execution/api-deployment.mdx @@ -215,6 +215,25 @@ while (true) { +#### Oversized outputs + +Workflow execution responses are capped by platform request and response limits. When an internal output, log field, streamed field, or async status payload contains a value that is too large to inline, Sim may replace that nested value with a versioned reference: + +```json +{ + "__simLargeValueRef": true, + "version": 1, + "id": "lv_abc123DEF456", + "kind": "array", + "size": 12582912, + "key": "execution/workspace-id/workflow-id/exec_xyz/large-value-lv_abc123DEF456.json", + "executionId": "exec_xyz", + "preview": { "length": 25000 } +} +``` + +The `version` field is part of the external API contract. Treat the reference as an opaque placeholder for a value that could not be safely embedded in the response. `id`, `key`, and `executionId` are not fetch URLs; `key` points to execution-scoped server storage. Use `selectedOutputs` to request a smaller nested field, reduce the data passed between blocks, or return the data from a Response block when your workflow intentionally owns the HTTP response body. File outputs are metadata-first; request `.base64` only when you need inline file content. JavaScript Function blocks can explicitly read large files or value refs with the `sim.files` and `sim.values` helpers under memory caps. + ### Asynchronous For long-running workflows, async mode returns a job ID immediately so you don't need to hold the connection open. Add the `X-Execution-Mode: async` header to your request. The API returns HTTP 202 with a job ID and status URL. Poll the status URL until the job completes. diff --git a/apps/docs/content/docs/en/introduction/index.mdx b/apps/docs/content/docs/en/introduction/index.mdx index 891667249f7..b9e783df711 100644 --- a/apps/docs/content/docs/en/introduction/index.mdx +++ b/apps/docs/content/docs/en/introduction/index.mdx @@ -10,13 +10,13 @@ import { FAQ } from '@/components/ui/faq' Sim is the open-source AI workspace where teams build, deploy, and manage AI agents. Create agents visually with the workflow builder, conversationally through Mothership, or programmatically with the API. Connect AI models, databases, APIs, and 1,000+ business tools to build agents that automate real work — from chatbots and compliance agents to data pipelines and ITSM automation. -
    +
    Sim visual workflow canvas
    diff --git a/apps/docs/content/docs/en/tools/grafana.mdx b/apps/docs/content/docs/en/tools/grafana.mdx index f3d0b63208a..47cbd9983d9 100644 --- a/apps/docs/content/docs/en/tools/grafana.mdx +++ b/apps/docs/content/docs/en/tools/grafana.mdx @@ -71,9 +71,11 @@ Search and list all dashboards | `organizationId` | string | No | Organization ID for multi-org Grafana instances \(e.g., 1, 2\) | | `query` | string | No | Search query to filter dashboards by title | | `tag` | string | No | Filter by tag \(comma-separated for multiple tags\) | -| `folderIds` | string | No | Filter by folder IDs \(comma-separated, e.g., 1,2,3\) | +| `folderUIDs` | string | No | Filter by folder UIDs \(comma-separated, e.g., abc123,def456\) | +| `dashboardUIDs` | string | No | Filter by dashboard UIDs \(comma-separated, e.g., abc123,def456\) | | `starred` | boolean | No | Only return starred dashboards | -| `limit` | number | No | Maximum number of dashboards to return | +| `limit` | number | No | Maximum number of dashboards to return \(default 1000\) | +| `page` | number | No | Page number for pagination \(1-based\) | #### Output @@ -136,7 +138,7 @@ Update an existing dashboard. Fetches the current dashboard and merges your chan | `timezone` | string | No | Dashboard timezone \(e.g., browser, utc\) | | `refresh` | string | No | Auto-refresh interval \(e.g., 5s, 1m, 5m\) | | `panels` | string | No | JSON array of panel configurations | -| `overwrite` | boolean | No | Overwrite even if there is a version conflict | +| `overwrite` | boolean | No | Overwrite even if there is a version conflict \(defaults to false to surface 412 conflicts\) | | `message` | string | No | Commit message for this version | #### Output @@ -188,13 +190,26 @@ List all alert rules in the Grafana instance | Parameter | Type | Description | | --------- | ---- | ----------- | | `rules` | array | List of alert rules | +| ↳ `id` | number | Alert rule numeric ID | | ↳ `uid` | string | Alert rule UID | | ↳ `title` | string | Alert rule title | -| ↳ `condition` | string | Alert condition | -| ↳ `folderUID` | string | Parent folder UID | -| ↳ `ruleGroup` | string | Rule group name | +| ↳ `condition` | string | RefId of the query used as the alert condition | +| ↳ `data` | json | Alert rule query/expression data array | +| ↳ `updated` | string | Last update timestamp | | ↳ `noDataState` | string | State when no data is returned | | ↳ `execErrState` | string | State on execution error | +| ↳ `for` | string | Duration the condition must hold before firing | +| ↳ `keepFiringFor` | string | Duration to keep firing after condition stops | +| ↳ `missingSeriesEvalsToResolve` | number | Number of missing series evaluations before resolving | +| ↳ `annotations` | json | Alert annotations | +| ↳ `labels` | json | Alert labels | +| ↳ `isPaused` | boolean | Whether the rule is paused | +| ↳ `folderUID` | string | Parent folder UID | +| ↳ `ruleGroup` | string | Rule group name | +| ↳ `orgID` | number | Organization ID | +| ↳ `provenance` | string | Provisioning source \(empty if API-managed\) | +| ↳ `notification_settings` | json | Per-rule notification settings \(overrides\) | +| ↳ `record` | json | Recording rule configuration \(recording rules only\) | ### `grafana_get_alert_rule` @@ -213,16 +228,26 @@ Get a specific alert rule by its UID | Parameter | Type | Description | | --------- | ---- | ----------- | +| `id` | number | Alert rule numeric ID | | `uid` | string | Alert rule UID | | `title` | string | Alert rule title | -| `condition` | string | Alert condition | -| `data` | json | Alert rule query data | -| `folderUID` | string | Parent folder UID | -| `ruleGroup` | string | Rule group name | +| `condition` | string | RefId of the query used as the alert condition | +| `data` | json | Alert rule query/expression data array | +| `updated` | string | Last update timestamp | | `noDataState` | string | State when no data is returned | | `execErrState` | string | State on execution error | +| `for` | string | Duration the condition must hold before firing | +| `keepFiringFor` | string | Duration to keep firing after condition stops | +| `missingSeriesEvalsToResolve` | number | Number of missing series evaluations before resolving | | `annotations` | json | Alert annotations | | `labels` | json | Alert labels | +| `isPaused` | boolean | Whether the rule is paused | +| `folderUID` | string | Parent folder UID | +| `ruleGroup` | string | Rule group name | +| `orgID` | number | Organization ID | +| `provenance` | string | Provisioning source \(empty if API-managed\) | +| `notification_settings` | json | Per-rule notification settings \(overrides\) | +| `record` | json | Recording rule configuration \(recording rules only\) | ### `grafana_create_alert_rule` @@ -238,22 +263,45 @@ Create a new alert rule | `title` | string | Yes | The title of the alert rule | | `folderUid` | string | Yes | The UID of the folder to create the alert in \(e.g., folder-abc123\) | | `ruleGroup` | string | Yes | The name of the rule group | -| `condition` | string | Yes | The refId of the query or expression to use as the alert condition | +| `condition` | string | No | The refId of the query or expression to use as the alert condition \(required for alerting rules; omit for recording rules\) | | `data` | string | Yes | JSON array of query/expression data objects | | `forDuration` | string | No | Duration to wait before firing \(e.g., 5m, 1h\) | | `noDataState` | string | No | State when no data is returned \(NoData, Alerting, OK\) | -| `execErrState` | string | No | State on execution error \(Alerting, OK\) | +| `execErrState` | string | No | State on execution error \(Error, Alerting, OK\) | | `annotations` | string | No | JSON object of annotations | | `labels` | string | No | JSON object of labels | +| `uid` | string | No | Optional custom UID for the alert rule | +| `isPaused` | boolean | No | Whether the rule is paused on creation | +| `keepFiringFor` | string | No | Duration to keep firing after the condition stops \(e.g., 5m\) | +| `missingSeriesEvalsToResolve` | number | No | Number of missing series evaluations before resolving | +| `notificationSettings` | string | No | JSON object of per-rule notification settings \(overrides\) | +| `record` | string | No | JSON object configuring this as a recording rule \(omit for alerting rules\) | +| `disableProvenance` | boolean | No | Set X-Disable-Provenance header so the rule remains editable in the Grafana UI | #### Output | Parameter | Type | Description | | --------- | ---- | ----------- | -| `uid` | string | The UID of the created alert rule | +| `id` | number | Alert rule numeric ID | +| `uid` | string | Alert rule UID | | `title` | string | Alert rule title | +| `condition` | string | RefId of the query used as the alert condition | +| `data` | json | Alert rule query/expression data array | +| `updated` | string | Last update timestamp | +| `noDataState` | string | State when no data is returned | +| `execErrState` | string | State on execution error | +| `for` | string | Duration the condition must hold before firing | +| `keepFiringFor` | string | Duration to keep firing after condition stops | +| `missingSeriesEvalsToResolve` | number | Number of missing series evaluations before resolving | +| `annotations` | json | Alert annotations | +| `labels` | json | Alert labels | +| `isPaused` | boolean | Whether the rule is paused | | `folderUID` | string | Parent folder UID | | `ruleGroup` | string | Rule group name | +| `orgID` | number | Organization ID | +| `provenance` | string | Provisioning source \(empty if API-managed\) | +| `notification_settings` | json | Per-rule notification settings \(overrides\) | +| `record` | json | Recording rule configuration \(recording rules only\) | ### `grafana_update_alert_rule` @@ -274,18 +322,40 @@ Update an existing alert rule. Fetches the current rule and merges your changes. | `data` | string | No | New JSON array of query/expression data objects | | `forDuration` | string | No | Duration to wait before firing \(e.g., 5m, 1h\) | | `noDataState` | string | No | State when no data is returned \(NoData, Alerting, OK\) | -| `execErrState` | string | No | State on execution error \(Alerting, OK\) | +| `execErrState` | string | No | State on execution error \(Error, Alerting, OK\) | | `annotations` | string | No | JSON object of annotations | | `labels` | string | No | JSON object of labels | +| `isPaused` | boolean | No | Whether the rule is paused | +| `keepFiringFor` | string | No | Duration to keep firing after the condition stops \(e.g., 5m\) | +| `missingSeriesEvalsToResolve` | number | No | Number of missing series evaluations before resolving | +| `notificationSettings` | string | No | JSON object of per-rule notification settings \(overrides\) | +| `record` | string | No | JSON object configuring this as a recording rule | +| `disableProvenance` | boolean | No | Set X-Disable-Provenance header so the rule remains editable in the Grafana UI | #### Output | Parameter | Type | Description | | --------- | ---- | ----------- | -| `uid` | string | The UID of the updated alert rule | +| `id` | number | Alert rule numeric ID | +| `uid` | string | Alert rule UID | | `title` | string | Alert rule title | +| `condition` | string | RefId of the query used as the alert condition | +| `data` | json | Alert rule query/expression data array | +| `updated` | string | Last update timestamp | +| `noDataState` | string | State when no data is returned | +| `execErrState` | string | State on execution error | +| `for` | string | Duration the condition must hold before firing | +| `keepFiringFor` | string | Duration to keep firing after condition stops | +| `missingSeriesEvalsToResolve` | number | Number of missing series evaluations before resolving | +| `annotations` | json | Alert annotations | +| `labels` | json | Alert labels | +| `isPaused` | boolean | Whether the rule is paused | | `folderUID` | string | Parent folder UID | | `ruleGroup` | string | Rule group name | +| `orgID` | number | Organization ID | +| `provenance` | string | Provisioning source \(empty if API-managed\) | +| `notification_settings` | json | Per-rule notification settings \(overrides\) | +| `record` | json | Recording rule configuration \(recording rules only\) | ### `grafana_delete_alert_rule` @@ -317,6 +387,7 @@ List all alert notification contact points | `apiKey` | string | Yes | Grafana Service Account Token | | `baseUrl` | string | Yes | Grafana instance URL \(e.g., https://your-grafana.com\) | | `organizationId` | string | No | Organization ID for multi-org Grafana instances \(e.g., 1, 2\) | +| `name` | string | No | Filter contact points by exact name match | #### Output @@ -327,6 +398,8 @@ List all alert notification contact points | ↳ `name` | string | Contact point name | | ↳ `type` | string | Notification type \(email, slack, etc.\) | | ↳ `settings` | object | Type-specific settings | +| ↳ `disableResolveMessage` | boolean | Whether resolve messages are disabled | +| ↳ `provenance` | string | Provisioning source \(empty if API-managed\) | ### `grafana_create_annotation` @@ -341,7 +414,7 @@ Create an annotation on a dashboard or as a global annotation | `organizationId` | string | No | Organization ID for multi-org Grafana instances \(e.g., 1, 2\) | | `text` | string | Yes | The text content of the annotation | | `tags` | string | No | Comma-separated list of tags | -| `dashboardUid` | string | Yes | UID of the dashboard to add the annotation to \(e.g., abc123def\) | +| `dashboardUid` | string | No | UID of the dashboard to add the annotation to \(e.g., abc123def\). Omit to create a global organization annotation. | | `panelId` | number | No | ID of the panel to add the annotation to \(e.g., 1, 2\) | | `time` | number | No | Start time in epoch milliseconds \(e.g., 1704067200000, defaults to now\) | | `timeEnd` | number | No | End time in epoch milliseconds for range annotations \(e.g., 1704153600000\) | @@ -366,8 +439,11 @@ Query annotations by time range, dashboard, or tags | `organizationId` | string | No | Organization ID for multi-org Grafana instances \(e.g., 1, 2\) | | `from` | number | No | Start time in epoch milliseconds \(e.g., 1704067200000\) | | `to` | number | No | End time in epoch milliseconds \(e.g., 1704153600000\) | -| `dashboardUid` | string | Yes | Dashboard UID to query annotations from \(e.g., abc123def\) | +| `dashboardUid` | string | No | Dashboard UID to query annotations from \(e.g., abc123def\). Omit to query annotations across the organization. | +| `dashboardId` | number | No | Legacy numeric dashboard ID filter \(prefer dashboardUid\) | | `panelId` | number | No | Filter by panel ID \(e.g., 1, 2\) | +| `alertId` | number | No | Filter by alert ID | +| `userId` | number | No | Filter by ID of the user who created the annotation | | `tags` | string | No | Comma-separated list of tags to filter by | | `type` | string | No | Filter by type \(alert or annotation\) | | `limit` | number | No | Maximum number of annotations to return | @@ -378,17 +454,19 @@ Query annotations by time range, dashboard, or tags | --------- | ---- | ----------- | | `annotations` | array | List of annotations | | ↳ `id` | number | Annotation ID | +| ↳ `alertId` | number | Associated alert ID \(0 if not alert-driven\) | | ↳ `dashboardId` | number | Dashboard ID | | ↳ `dashboardUID` | string | Dashboard UID | -| ↳ `created` | number | Creation timestamp in epoch ms | -| ↳ `updated` | number | Last update timestamp in epoch ms | +| ↳ `panelId` | number | Panel ID within the dashboard | +| ↳ `userId` | number | ID of the user who created the annotation | +| ↳ `userName` | string | Username of the user who created the annotation | +| ↳ `newState` | string | New alert state \(alert annotations only\) | +| ↳ `prevState` | string | Previous alert state \(alert annotations only\) | | ↳ `time` | number | Start time in epoch ms | | ↳ `timeEnd` | number | End time in epoch ms | | ↳ `text` | string | Annotation text | +| ↳ `metric` | string | Metric associated with the annotation | | ↳ `tags` | array | Annotation tags | -| ↳ `login` | string | Login of the user who created the annotation | -| ↳ `email` | string | Email of the user who created the annotation | -| ↳ `avatarUrl` | string | Avatar URL of the user | | ↳ `data` | json | Additional annotation data object from Grafana | ### `grafana_update_annotation` @@ -403,7 +481,7 @@ Update an existing annotation | `baseUrl` | string | Yes | Grafana instance URL \(e.g., https://your-grafana.com\) | | `organizationId` | string | No | Organization ID for multi-org Grafana instances \(e.g., 1, 2\) | | `annotationId` | number | Yes | The ID of the annotation to update | -| `text` | string | Yes | New text content for the annotation | +| `text` | string | No | New text content for the annotation \(PATCH supports partial updates\) | | `tags` | string | No | Comma-separated list of new tags | | `time` | number | No | New start time in epoch milliseconds \(e.g., 1704067200000\) | | `timeEnd` | number | No | New end time in epoch milliseconds \(e.g., 1704153600000\) | @@ -453,10 +531,22 @@ List all data sources configured in Grafana | `dataSources` | array | List of data sources | | ↳ `id` | number | Data source ID | | ↳ `uid` | string | Data source UID | +| ↳ `orgId` | number | Organization ID | | ↳ `name` | string | Data source name | | ↳ `type` | string | Data source type \(prometheus, mysql, etc.\) | +| ↳ `typeLogoUrl` | string | Logo URL for the data source type | +| ↳ `access` | string | Access mode \(proxy or direct\) | | ↳ `url` | string | Data source URL | +| ↳ `user` | string | Username used to connect | +| ↳ `database` | string | Database name \(if applicable\) | +| ↳ `basicAuth` | boolean | Whether basic auth is enabled | +| ↳ `basicAuthUser` | string | Basic auth username | +| ↳ `withCredentials` | boolean | Whether to send credentials with cross-origin requests | | ↳ `isDefault` | boolean | Whether this is the default data source | +| ↳ `jsonData` | object | Type-specific JSON configuration | +| ↳ `secureJsonFields` | object | Map of secure fields that are set \(values are not returned\) | +| ↳ `version` | number | Data source version | +| ↳ `readOnly` | boolean | Whether the data source is read-only | ### `grafana_get_data_source` @@ -477,12 +567,22 @@ Get a data source by its ID or UID | --------- | ---- | ----------- | | `id` | number | Data source ID | | `uid` | string | Data source UID | +| `orgId` | number | Organization ID | | `name` | string | Data source name | | `type` | string | Data source type | +| `typeLogoUrl` | string | Logo URL for the data source type | +| `access` | string | Access mode \(proxy or direct\) | | `url` | string | Data source connection URL | +| `user` | string | Username used to connect | | `database` | string | Database name \(if applicable\) | +| `basicAuth` | boolean | Whether basic auth is enabled | +| `basicAuthUser` | string | Basic auth username | +| `withCredentials` | boolean | Whether to send credentials with cross-origin requests | | `isDefault` | boolean | Whether this is the default data source | | `jsonData` | json | Additional data source configuration | +| `secureJsonFields` | object | Map of secure fields that are set \(values are not returned\) | +| `version` | number | Data source version | +| `readOnly` | boolean | Whether the data source is read-only | ### `grafana_list_folders` @@ -497,6 +597,7 @@ List all folders in Grafana | `organizationId` | string | No | Organization ID for multi-org Grafana instances \(e.g., 1, 2\) | | `limit` | number | No | Maximum number of folders to return | | `page` | number | No | Page number for pagination | +| `parentUid` | string | No | List children of this folder UID \(requires nested folders enabled\) | #### Output @@ -506,16 +607,18 @@ List all folders in Grafana | ↳ `id` | number | Folder ID | | ↳ `uid` | string | Folder UID | | ↳ `title` | string | Folder title | +| ↳ `url` | string | Folder URL path | +| ↳ `parentUid` | string | Parent folder UID \(nested folders only\) | +| ↳ `parents` | array | Ancestor folder hierarchy \(nested folders only\) | | ↳ `hasAcl` | boolean | Whether the folder has custom ACL permissions | | ↳ `canSave` | boolean | Whether the current user can save the folder | | ↳ `canEdit` | boolean | Whether the current user can edit the folder | | ↳ `canAdmin` | boolean | Whether the current user has admin rights | -| ↳ `canDelete` | boolean | Whether the current user can delete the folder | | ↳ `createdBy` | string | Username of who created the folder | | ↳ `created` | string | Timestamp when the folder was created | | ↳ `updatedBy` | string | Username of who last updated the folder | | ↳ `updated` | string | Timestamp when the folder was last updated | -| ↳ `version` | number | Version number of the folder | +| ↳ `version` | number | Folder version number | ### `grafana_create_folder` @@ -530,6 +633,7 @@ Create a new folder in Grafana | `organizationId` | string | No | Organization ID for multi-org Grafana instances \(e.g., 1, 2\) | | `title` | string | Yes | The title of the new folder | | `uid` | string | No | Optional UID for the folder \(auto-generated if not provided\) | +| `parentUid` | string | No | Parent folder UID for nested folders \(requires nested folders enabled\) | #### Output @@ -539,11 +643,12 @@ Create a new folder in Grafana | `uid` | string | The UID of the created folder | | `title` | string | The title of the created folder | | `url` | string | The URL path to the folder | +| `parentUid` | string | Parent folder UID \(nested folders only\) | +| `parents` | array | Ancestor folder hierarchy \(nested folders only\) | | `hasAcl` | boolean | Whether the folder has custom ACL permissions | | `canSave` | boolean | Whether the current user can save the folder | | `canEdit` | boolean | Whether the current user can edit the folder | | `canAdmin` | boolean | Whether the current user has admin rights on the folder | -| `canDelete` | boolean | Whether the current user can delete the folder | | `createdBy` | string | Username of who created the folder | | `created` | string | Timestamp when the folder was created | | `updatedBy` | string | Username of who last updated the folder | diff --git a/apps/docs/content/docs/es/api-reference/getting-started.mdx b/apps/docs/content/docs/es/api-reference/getting-started.mdx index dced7aca61c..038998853cf 100644 --- a/apps/docs/content/docs/es/api-reference/getting-started.mdx +++ b/apps/docs/content/docs/es/api-reference/getting-started.mdx @@ -109,20 +109,22 @@ curl -X POST https://www.sim.ai/api/workflows/{workflowId}/execute \ -d '{"inputs": {}, "async": true}' ``` -This returns immediately with a `taskId`: +This returns immediately with a `jobId` and `statusUrl`: ```json { "success": true, - "taskId": "job_abc123", - "status": "queued" + "jobId": "job_abc123", + "statusUrl": "https://www.sim.ai/api/jobs/job_abc123", + "message": "Workflow execution started", + "async": true } ``` Poll the [Get Job Status](/api-reference/workflows/getJobStatus) endpoint until the status is `completed` or `failed`: ```bash -curl https://www.sim.ai/api/jobs/{taskId} \ +curl https://www.sim.ai/api/jobs/{jobId} \ -H "X-API-Key: YOUR_API_KEY" ``` diff --git a/apps/docs/content/docs/fr/api-reference/getting-started.mdx b/apps/docs/content/docs/fr/api-reference/getting-started.mdx index dced7aca61c..038998853cf 100644 --- a/apps/docs/content/docs/fr/api-reference/getting-started.mdx +++ b/apps/docs/content/docs/fr/api-reference/getting-started.mdx @@ -109,20 +109,22 @@ curl -X POST https://www.sim.ai/api/workflows/{workflowId}/execute \ -d '{"inputs": {}, "async": true}' ``` -This returns immediately with a `taskId`: +This returns immediately with a `jobId` and `statusUrl`: ```json { "success": true, - "taskId": "job_abc123", - "status": "queued" + "jobId": "job_abc123", + "statusUrl": "https://www.sim.ai/api/jobs/job_abc123", + "message": "Workflow execution started", + "async": true } ``` Poll the [Get Job Status](/api-reference/workflows/getJobStatus) endpoint until the status is `completed` or `failed`: ```bash -curl https://www.sim.ai/api/jobs/{taskId} \ +curl https://www.sim.ai/api/jobs/{jobId} \ -H "X-API-Key: YOUR_API_KEY" ``` diff --git a/apps/docs/content/docs/ja/api-reference/getting-started.mdx b/apps/docs/content/docs/ja/api-reference/getting-started.mdx index dced7aca61c..038998853cf 100644 --- a/apps/docs/content/docs/ja/api-reference/getting-started.mdx +++ b/apps/docs/content/docs/ja/api-reference/getting-started.mdx @@ -109,20 +109,22 @@ curl -X POST https://www.sim.ai/api/workflows/{workflowId}/execute \ -d '{"inputs": {}, "async": true}' ``` -This returns immediately with a `taskId`: +This returns immediately with a `jobId` and `statusUrl`: ```json { "success": true, - "taskId": "job_abc123", - "status": "queued" + "jobId": "job_abc123", + "statusUrl": "https://www.sim.ai/api/jobs/job_abc123", + "message": "Workflow execution started", + "async": true } ``` Poll the [Get Job Status](/api-reference/workflows/getJobStatus) endpoint until the status is `completed` or `failed`: ```bash -curl https://www.sim.ai/api/jobs/{taskId} \ +curl https://www.sim.ai/api/jobs/{jobId} \ -H "X-API-Key: YOUR_API_KEY" ``` diff --git a/apps/docs/content/docs/zh/api-reference/getting-started.mdx b/apps/docs/content/docs/zh/api-reference/getting-started.mdx index dced7aca61c..038998853cf 100644 --- a/apps/docs/content/docs/zh/api-reference/getting-started.mdx +++ b/apps/docs/content/docs/zh/api-reference/getting-started.mdx @@ -109,20 +109,22 @@ curl -X POST https://www.sim.ai/api/workflows/{workflowId}/execute \ -d '{"inputs": {}, "async": true}' ``` -This returns immediately with a `taskId`: +This returns immediately with a `jobId` and `statusUrl`: ```json { "success": true, - "taskId": "job_abc123", - "status": "queued" + "jobId": "job_abc123", + "statusUrl": "https://www.sim.ai/api/jobs/job_abc123", + "message": "Workflow execution started", + "async": true } ``` Poll the [Get Job Status](/api-reference/workflows/getJobStatus) endpoint until the status is `completed` or `failed`: ```bash -curl https://www.sim.ai/api/jobs/{taskId} \ +curl https://www.sim.ai/api/jobs/{jobId} \ -H "X-API-Key: YOUR_API_KEY" ``` diff --git a/apps/docs/content/docs/zh/api-reference/python.mdx b/apps/docs/content/docs/zh/api-reference/python.mdx index c44973c8660..608942d1baf 100644 --- a/apps/docs/content/docs/zh/api-reference/python.mdx +++ b/apps/docs/content/docs/zh/api-reference/python.mdx @@ -117,20 +117,20 @@ if is_ready: 获取异步任务执行的状态。 ```python -status = client.get_job_status("task-id-from-async-execution") +status = client.get_job_status("job-id-from-async-execution") print("Status:", status["status"]) # 'queued', 'processing', 'completed', 'failed' if status["status"] == "completed": print("Output:", status["output"]) ``` **参数：** -- `task_id` (str): 异步执行返回的任务 ID +- `job_id` (str): 异步执行返回的作业 ID **返回值：** `Dict[str, Any]` **响应字段：** - `success` (bool): 请求是否成功 -- `taskId` (str): 任务 ID +- `taskId` (str): 作业 ID - `status` (str): 可能的值包括 `'queued'`, `'processing'`, `'completed'`, `'failed'`, `'cancelled'` - `metadata` (dict): 包含 `startedAt`, `completedAt` 和 `duration` - `output` (any, optional): 工作流输出（完成时） @@ -271,10 +271,11 @@ class WorkflowExecutionResult: @dataclass class AsyncExecutionResult: success: bool - task_id: str - status: str # 'queued' - created_at: str - links: Dict[str, str] # e.g., {"status": "/api/jobs/{taskId}"} + job_id: str + status_url: str + execution_id: Optional[str] = None + message: str = "" + async_execution: bool = True ``` ### WorkflowStatus @@ -494,17 +495,17 @@ def execute_async(): ) # Check if result is an async execution - if hasattr(result, 'task_id'): - print(f"Task ID: {result.task_id}") - print(f"Status endpoint: {result.links['status']}") + if hasattr(result, 'job_id'): + print(f"Job ID: {result.job_id}") + print(f"Status endpoint: {result.status_url}") # Poll for completion - status = client.get_job_status(result.task_id) + status = client.get_job_status(result.job_id) while status["status"] in ["queued", "processing"]: print(f"Current status: {status['status']}") time.sleep(2) # Wait 2 seconds - status = client.get_job_status(result.task_id) + status = client.get_job_status(result.job_id) if status["status"] == "completed": print("Workflow completed!") diff --git a/apps/docs/content/docs/zh/api-reference/typescript.mdx b/apps/docs/content/docs/zh/api-reference/typescript.mdx index 0f038db92dd..fac3bdffb73 100644 --- a/apps/docs/content/docs/zh/api-reference/typescript.mdx +++ b/apps/docs/content/docs/zh/api-reference/typescript.mdx @@ -138,7 +138,7 @@ if (isReady) { 获取异步任务执行的状态。 ```typescript -const status = await client.getJobStatus('task-id-from-async-execution'); +const status = await client.getJobStatus('job-id-from-async-execution'); console.log('Status:', status.status); // 'queued', 'processing', 'completed', 'failed' if (status.status === 'completed') { console.log('Output:', status.output); @@ -146,13 +146,13 @@ if (status.status === 'completed') { ``` **参数：** -- `taskId`（字符串）：异步执行返回的任务 ID +- `jobId`（字符串）：异步执行返回的作业 ID **返回值：** `Promise` **响应字段：** - `success`（布尔值）：请求是否成功 -- `taskId`（字符串）：任务 ID +- `taskId`（字符串）：作业 ID - `status`（字符串）：以下之一 `'queued'`、`'processing'`、`'completed'`、`'failed'`、`'cancelled'` - `metadata`（对象）：包含 `startedAt`、`completedAt` 和 `duration` - `output`（任意类型，可选）：工作流输出（完成时） @@ -286,12 +286,11 @@ interface WorkflowExecutionResult { ```typescript interface AsyncExecutionResult { success: boolean; - taskId: string; - status: 'queued'; - createdAt: string; - links: { - status: string; // e.g., "/api/jobs/{taskId}" - }; + jobId: string; + statusUrl: string; + executionId?: string; + message: string; + async: true; } ``` @@ -797,17 +796,17 @@ async function executeAsync() { }); // Check if result is an async execution - if ('taskId' in result) { - console.log('Task ID:', result.taskId); - console.log('Status endpoint:', result.links.status); + if ('jobId' in result) { + console.log('Job ID:', result.jobId); + console.log('Status endpoint:', result.statusUrl); // Poll for completion - let status = await client.getJobStatus(result.taskId); + let status = await client.getJobStatus(result.jobId); while (status.status === 'queued' || status.status === 'processing') { console.log('Current status:', status.status); await new Promise(resolve => setTimeout(resolve, 2000)); // Wait 2 seconds - status = await client.getJobStatus(result.taskId); + status = await client.getJobStatus(result.jobId); } if (status.status === 'completed') { diff --git a/apps/realtime/src/database/operations.ts b/apps/realtime/src/database/operations.ts index 14fa8639eaf..38a98b14bb3 100644 --- a/apps/realtime/src/database/operations.ts +++ b/apps/realtime/src/database/operations.ts @@ -40,6 +40,7 @@ const db = socketDb const DEFAULT_LOOP_ITERATIONS = 5 const DEFAULT_PARALLEL_COUNT = 5 +const DEFAULT_PARALLEL_BATCH_SIZE = 20 /** Minimal block shape needed for protection and descendant checks */ interface DbBlockRef { @@ -740,8 +741,9 @@ async function handleBlocksOperationTx( workflowId, type: 'parallel', config: { - parallelType: 'fixed', + parallelType: 'count', count: DEFAULT_PARALLEL_COUNT, + batchSize: DEFAULT_PARALLEL_BATCH_SIZE, nodes: [], }, }) @@ -1620,11 +1622,23 @@ async function handleSubflowOperationTx( logger.debug(`Updating subflow ${payload.id} with config:`, payload.config) - // Update the subflow configuration + // Read-modify-write merge so partial config payloads never wipe other fields + // (e.g. an iteration-only update from one client should not drop batchSize set by another) + const existingSubflow = await tx + .select({ config: workflowSubflows.config }) + .from(workflowSubflows) + .where( + and(eq(workflowSubflows.id, payload.id), eq(workflowSubflows.workflowId, workflowId)) + ) + .limit(1) + + const existingConfig = (existingSubflow[0]?.config as Record) || {} + const mergedConfig = { ...existingConfig, ...payload.config } + const updateResult = await tx .update(workflowSubflows) .set({ - config: payload.config, + config: mergedConfig, updatedAt: new Date(), }) .where( @@ -1677,27 +1691,35 @@ async function handleSubflowOperationTx( }) .where(and(eq(workflowBlocks.id, payload.id), eq(workflowBlocks.workflowId, workflowId))) } else if (payload.type === 'parallel') { - // Update the parallel block's data properties - const blockData = { - ...payload.config, - width: 500, - height: 300, - type: 'subflowNode', - } - - // Include count if provided - if (payload.config.count !== undefined) { - blockData.count = payload.config.count - } + const existingBlock = await tx + .select({ data: workflowBlocks.data }) + .from(workflowBlocks) + .where(and(eq(workflowBlocks.id, payload.id), eq(workflowBlocks.workflowId, workflowId))) + .limit(1) - // Include collection if provided - if (payload.config.distribution !== undefined) { - blockData.collection = payload.config.distribution - } + const existingData = (existingBlock[0]?.data as any) || {} - // Include parallelType if provided - if (payload.config.parallelType !== undefined) { - blockData.parallelType = payload.config.parallelType + const blockData: any = { + ...existingData, + type: 'subflowNode', + width: existingData.width ?? 500, + height: existingData.height ?? 300, + count: + payload.config.count !== undefined + ? payload.config.count + : (existingData.count ?? DEFAULT_PARALLEL_COUNT), + parallelType: + payload.config.parallelType !== undefined + ? payload.config.parallelType + : (existingData.parallelType ?? 'count'), + collection: + payload.config.distribution !== undefined + ? payload.config.distribution + : (existingData.collection ?? ''), + batchSize: + payload.config.batchSize !== undefined + ? payload.config.batchSize + : (existingData.batchSize ?? DEFAULT_PARALLEL_BATCH_SIZE), } await tx diff --git a/apps/sim/app/api/auth/oauth/wealthbox/items/route.ts b/apps/sim/app/api/auth/oauth/wealthbox/items/route.ts index 102e8f16c08..6a31bcf3b9f 100644 --- a/apps/sim/app/api/auth/oauth/wealthbox/items/route.ts +++ b/apps/sim/app/api/auth/oauth/wealthbox/items/route.ts @@ -1,14 +1,12 @@ -import { db } from '@sim/db' -import { account } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { wealthboxOAuthItemsContract } from '@/lib/api/contracts/selectors/wealthbox' import { parseRequest } from '@/lib/api/server' -import { getSession } from '@/lib/auth' +import { authorizeCredentialUse } from '@/lib/auth/credential-access' +import { validatePathSegment } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded, resolveOAuthAccountId } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' export const dynamic = 'force-dynamic' @@ -30,51 +28,34 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { - const session = await getSession() - - if (!session?.user?.id) { - logger.warn(`[${requestId}] Unauthenticated request rejected`) - return NextResponse.json({ error: 'User not authenticated' }, { status: 401 }) - } - const parsed = await parseRequest(wealthboxOAuthItemsContract, request, {}) if (!parsed.success) return parsed.response const { credentialId, type } = parsed.data.query const query = parsed.data.query.query ?? '' - const resolved = await resolveOAuthAccountId(credentialId) - if (!resolved) { - return NextResponse.json({ error: 'Credential not found' }, { status: 404 }) - } - - if (resolved.workspaceId) { - const { getUserEntityPermissions } = await import('@/lib/workspaces/permissions/utils') - const perm = await getUserEntityPermissions( - session.user.id, - 'workspace', - resolved.workspaceId - ) - if (perm === null) { - return NextResponse.json({ error: 'Forbidden' }, { status: 403 }) - } + const credentialIdValidation = validatePathSegment(credentialId, { + paramName: 'credentialId', + maxLength: 100, + allowHyphens: true, + allowUnderscores: true, + allowDots: false, + }) + if (!credentialIdValidation.isValid) { + logger.warn(`[${requestId}] Invalid credentialId format: ${credentialId}`) + return NextResponse.json({ error: credentialIdValidation.error }, { status: 400 }) } - const credentials = await db - .select() - .from(account) - .where(eq(account.id, resolved.accountId)) - .limit(1) - - if (!credentials.length) { - logger.warn(`[${requestId}] Credential not found`, { credentialId }) - return NextResponse.json({ error: 'Credential not found' }, { status: 404 }) + const authz = await authorizeCredentialUse(request, { + credentialId, + requireWorkflowIdForInternal: false, + }) + if (!authz.ok || !authz.credentialOwnerUserId) { + return NextResponse.json({ error: authz.error || 'Unauthorized' }, { status: 403 }) } - const accountRow = credentials[0] - const accessToken = await refreshAccessTokenIfNeeded( - resolved.accountId, - accountRow.userId, + credentialId, + authz.credentialOwnerUserId, requestId ) diff --git a/apps/sim/app/api/chat/[identifier]/route.ts b/apps/sim/app/api/chat/[identifier]/route.ts index a6dff447355..f35d950a21c 100644 --- a/apps/sim/app/api/chat/[identifier]/route.ts +++ b/apps/sim/app/api/chat/[identifier]/route.ts @@ -274,6 +274,9 @@ export const POST = withRouteHandler( workflowTriggerType: 'chat', }, executionId, + workspaceId, + workflowId: deployment.workflowId, + userId: workspaceOwnerId, executeFn: async ({ onStream, onBlockComplete, abortSignal }) => executeWorkflow( workflowForExecution, diff --git a/apps/sim/app/api/files/authorization.ts b/apps/sim/app/api/files/authorization.ts index a6fffe1d41b..ef5183ae0da 100644 --- a/apps/sim/app/api/files/authorization.ts +++ b/apps/sim/app/api/files/authorization.ts @@ -2,6 +2,7 @@ import { db } from '@sim/db' import { document, knowledgeBase, workspaceFile } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { and, eq, isNull, like, or } from 'drizzle-orm' +import { NextResponse } from 'next/server' import { getFileMetadata } from '@/lib/uploads' import type { StorageContext } from '@/lib/uploads/config' import { BLOB_CHAT_CONFIG, S3_CHAT_CONFIG } from '@/lib/uploads/config' @@ -587,6 +588,36 @@ async function authorizeFileAccess( } } +/** + * Guard helper for tool routes that download user files from storage. + * + * Validates that `key` is a non-empty string, that `userId` is present, and + * that the authenticated user owns the file. Returns a 404 `NextResponse` on + * any failure so callers can `return` it immediately; returns `null` when + * access is granted. + */ +export async function assertToolFileAccess( + key: unknown, + userId: string | undefined, + requestId: string, + routeLogger: ReturnType +): Promise { + if (typeof key !== 'string' || key.length === 0) { + routeLogger.warn(`[${requestId}] File access check rejected: missing key`) + return NextResponse.json({ success: false, error: 'File not found' }, { status: 404 }) + } + if (!userId) { + routeLogger.warn(`[${requestId}] File access check requires userId but none available`) + return NextResponse.json({ success: false, error: 'File not found' }, { status: 404 }) + } + const hasAccess = await verifyFileAccess(key, userId) + if (!hasAccess) { + routeLogger.warn(`[${requestId}] File access denied for user`, { userId, key }) + return NextResponse.json({ success: false, error: 'File not found' }, { status: 404 }) + } + return null +} + /** * Get chat storage configuration based on current storage provider */ diff --git a/apps/sim/app/api/files/multipart/route.ts b/apps/sim/app/api/files/multipart/route.ts index e61cbd543a8..836fc40ad6b 100644 --- a/apps/sim/app/api/files/multipart/route.ts +++ b/apps/sim/app/api/files/multipart/route.ts @@ -136,7 +136,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const config = getStorageConfig(storageContext) let customKey: string | undefined - if (context === 'workspace') { + if (context === 'workspace' || context === 'mothership') { const { MAX_WORKSPACE_FILE_SIZE } = await import('@/lib/uploads/shared/types') if (typeof fileSize === 'number' && fileSize > MAX_WORKSPACE_FILE_SIZE) { return NextResponse.json( @@ -158,11 +158,6 @@ export const POST = withRouteHandler(async (request: NextRequest) => { { status: 413 } ) } - } else if (context === 'mothership') { - const { generateWorkspaceFileKey } = await import( - '@/lib/uploads/contexts/workspace/workspace-file-manager' - ) - customKey = generateWorkspaceFileKey(workspaceId, fileName) } else if (context === 'execution') { const workflowId = (data as { workflowId?: unknown }).workflowId const executionId = (data as { executionId?: unknown }).executionId diff --git a/apps/sim/app/api/form/[identifier]/route.ts b/apps/sim/app/api/form/[identifier]/route.ts index b91c6ef932a..d5ed51c4af7 100644 --- a/apps/sim/app/api/form/[identifier]/route.ts +++ b/apps/sim/app/api/form/[identifier]/route.ts @@ -227,6 +227,9 @@ export const POST = withRouteHandler( workflowTriggerType: 'api', }, executionId, + workspaceId, + workflowId: deployment.workflowId, + userId: workspaceOwnerId, executeFn: async ({ onStream, onBlockComplete, abortSignal }) => executeWorkflow( workflowForExecution, diff --git a/apps/sim/app/api/form/manage/[id]/route.ts b/apps/sim/app/api/form/manage/[id]/route.ts index 242d923d306..d1c05bbe627 100644 --- a/apps/sim/app/api/form/manage/[id]/route.ts +++ b/apps/sim/app/api/form/manage/[id]/route.ts @@ -197,9 +197,12 @@ export const DELETE = withRouteHandler( return createErrorResponse('Form not found or access denied', 404) } - await db.delete(form).where(eq(form.id, id)) + await db + .update(form) + .set({ archivedAt: new Date(), isActive: false, updatedAt: new Date() }) + .where(eq(form.id, id)) - logger.info(`Form ${id} deleted (soft delete)`) + logger.info(`Form ${id} soft deleted`) recordAudit({ workspaceId: formWorkspaceId ?? null, diff --git a/apps/sim/app/api/function/execute/route.ts b/apps/sim/app/api/function/execute/route.ts index fcfda730c4b..1b2d5f844e1 100644 --- a/apps/sim/app/api/function/execute/route.ts +++ b/apps/sim/app/api/function/execute/route.ts @@ -12,8 +12,18 @@ import { isE2bEnabled } from '@/lib/core/config/feature-flags' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { executeInE2B, executeShellInE2B } from '@/lib/execution/e2b' -import { executeInIsolatedVM } from '@/lib/execution/isolated-vm' +import { executeInIsolatedVM, type IsolatedVMBrokerHandler } from '@/lib/execution/isolated-vm' import { CodeLanguage, DEFAULT_CODE_LANGUAGE, isValidCodeLanguage } from '@/lib/execution/languages' +import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref' +import { + MAX_FUNCTION_INLINE_BYTES, + MAX_INLINE_MATERIALIZATION_BYTES, + readUserFileContent, + unavailableLargeValueError, +} from '@/lib/execution/payloads/materialization.server' +import { compactExecutionPayload } from '@/lib/execution/payloads/serializer' +import { materializeLargeValueRef } from '@/lib/execution/payloads/store' +import { isExecutionResourceLimitError } from '@/lib/execution/resource-errors' import { uploadWorkspaceFile } from '@/lib/uploads/contexts/workspace/workspace-file-manager' import { getWorkflowById } from '@/lib/workflows/utils' import { escapeRegExp, normalizeName, REFERENCE } from '@/executor/constants' @@ -684,6 +694,125 @@ function serializeForShellEnv(value: unknown, nullValue = ''): string { } } +interface FunctionRouteExecutionContext { + workflowId?: string + workspaceId?: string + executionId?: string + largeValueExecutionIds?: string[] + allowLargeValueWorkflowScope?: boolean + userId?: string + requestId: string +} + +function asRecord(value: unknown): Record { + return value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : {} +} + +function getPositiveNumber(value: unknown): number | undefined { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) { + return undefined + } + return value +} + +function clampInlineBytes(value: unknown, limit = MAX_FUNCTION_INLINE_BYTES): number { + const requested = getPositiveNumber(value) + return Math.min(requested ?? limit, limit) +} + +function getBrokerFileArgs(args: unknown): { + file: unknown + maxBytes: number + offset?: number + length?: number +} { + const record = asRecord(args) + const options = asRecord(record.options) + return { + file: record.file, + maxBytes: clampInlineBytes(options.maxBytes), + offset: getPositiveNumber(options.offset), + length: getPositiveNumber(options.length), + } +} + +function createFunctionRuntimeBrokers( + context: FunctionRouteExecutionContext +): Record { + const base = { + requestId: context.requestId, + workflowId: context.workflowId, + workspaceId: context.workspaceId, + executionId: context.executionId, + largeValueExecutionIds: context.largeValueExecutionIds, + allowLargeValueWorkflowScope: context.allowLargeValueWorkflowScope, + userId: context.userId, + logger, + } + + const readFile = async (args: unknown, encoding: 'base64' | 'text', chunked = false) => { + const fileArgs = getBrokerFileArgs(args) + return readUserFileContent(fileArgs.file, { + ...base, + encoding, + maxBytes: fileArgs.maxBytes, + chunked, + offset: chunked ? fileArgs.offset : undefined, + length: chunked ? fileArgs.length : undefined, + }) + } + + return { + 'sim.files.readBase64': (args) => readFile(args, 'base64'), + 'sim.files.readText': (args) => readFile(args, 'text'), + 'sim.files.readBase64Chunk': (args) => readFile(args, 'base64', true), + 'sim.files.readTextChunk': (args) => readFile(args, 'text', true), + 'sim.values.read': async (args) => { + const record = asRecord(args) + const options = asRecord(record.options) + const ref = record.ref + if (!isLargeValueRef(ref)) { + throw new Error('Expected a large execution value reference.') + } + if (!context.executionId) { + throw new Error('Large execution values require an execution context.') + } + const value = await materializeLargeValueRef(ref, { + ...base, + maxBytes: clampInlineBytes(options.maxBytes, MAX_INLINE_MATERIALIZATION_BYTES), + }) + if (value === undefined) { + throw unavailableLargeValueError(ref) + } + return value + }, + } +} + +async function compactFunctionRouteBody( + body: T, + context: FunctionRouteExecutionContext +): Promise { + return compactExecutionPayload(body, { + workflowId: context.workflowId, + workspaceId: context.workspaceId, + executionId: context.executionId, + userId: context.userId, + preserveRoot: true, + requireDurable: Boolean(context.workspaceId && context.workflowId && context.executionId), + }) +} + +async function functionJsonResponse( + body: T, + context: FunctionRouteExecutionContext, + init?: ResponseInit +) { + return NextResponse.json(await compactFunctionRouteBody(body, context), init) +} + async function maybeExportSandboxFileToWorkspace(args: { authUserId: string workflowId?: string @@ -792,6 +921,7 @@ export const POST = withRouteHandler(async (req: NextRequest) => { let userCodeStartLine = 3 // Default value for error reporting let resolvedCode = '' // Store resolved code for error reporting let sourceCodeForErrors: string | undefined + let routeContext: FunctionRouteExecutionContext | undefined try { const auth = await checkInternalAuth(req) @@ -823,6 +953,9 @@ export const POST = withRouteHandler(async (req: NextRequest) => { workflowVariables = {}, contextVariables: preResolvedContextVariables = {}, workflowId, + executionId, + largeValueExecutionIds, + allowLargeValueWorkflowScope = false, workspaceId, isCustomTool = false, _sandboxFiles, @@ -837,9 +970,20 @@ export const POST = withRouteHandler(async (req: NextRequest) => { paramsCount: Object.keys(executionParams).length, timeout, workflowId, + executionId, isCustomTool, }) + routeContext = { + workflowId, + workspaceId, + executionId, + largeValueExecutionIds, + allowLargeValueWorkflowScope, + userId: auth.userId, + requestId, + } + const lang = isValidCodeLanguage(language) ? language : DEFAULT_CODE_LANGUAGE let contextVariables: Record = {} @@ -927,12 +1071,13 @@ export const POST = withRouteHandler(async (req: NextRequest) => { }) if (shellError) { - return NextResponse.json( + return functionJsonResponse( { success: false, error: shellError, output: { result: null, stdout: cleanStdout(shellStdout), executionTime }, }, + routeContext, { status: 500 } ) } @@ -953,10 +1098,13 @@ export const POST = withRouteHandler(async (req: NextRequest) => { if (fileExportResponse) return fileExportResponse } - return NextResponse.json({ - success: true, - output: { result: shellResult ?? null, stdout: cleanStdout(shellStdout), executionTime }, - }) + return functionJsonResponse( + { + success: true, + output: { result: shellResult ?? null, stdout: cleanStdout(shellStdout), executionTime }, + }, + routeContext + ) } if (lang === CodeLanguage.Python && !isE2bEnabled) { @@ -1054,12 +1202,13 @@ export const POST = withRouteHandler(async (req: NextRequest) => { errorDisplayCode, prologueLineCount + importLineCount ) - return NextResponse.json( + return functionJsonResponse( { success: false, error: formattedError, output: { result: null, stdout: cleanedOutput, executionTime }, }, + routeContext, { status: 500 } ) } @@ -1080,10 +1229,13 @@ export const POST = withRouteHandler(async (req: NextRequest) => { if (fileExportResponse) return fileExportResponse } - return NextResponse.json({ - success: true, - output: { result: e2bResult ?? null, stdout: cleanStdout(stdout), executionTime }, - }) + return functionJsonResponse( + { + success: true, + output: { result: e2bResult ?? null, stdout: cleanStdout(stdout), executionTime }, + }, + routeContext + ) } let prologueLineCount = 0 @@ -1137,12 +1289,13 @@ export const POST = withRouteHandler(async (req: NextRequest) => { errorDisplayCode, prologueLineCount ) - return NextResponse.json( + return functionJsonResponse( { success: false, error: formattedError, output: { result: null, stdout: cleanedOutput, executionTime }, }, + routeContext, { status: 500 } ) } @@ -1163,10 +1316,13 @@ export const POST = withRouteHandler(async (req: NextRequest) => { if (fileExportResponse) return fileExportResponse } - return NextResponse.json({ - success: true, - output: { result: e2bResult ?? null, stdout: cleanStdout(stdout), executionTime }, - }) + return functionJsonResponse( + { + success: true, + output: { result: e2bResult ?? null, stdout: cleanStdout(stdout), executionTime }, + }, + routeContext + ) } const executionMethod = 'isolated-vm' @@ -1194,16 +1350,19 @@ export const POST = withRouteHandler(async (req: NextRequest) => { prependedLineCount = paramKeys.length } - const isolatedResult = await executeInIsolatedVM({ - code: codeToExecute, - params: executionParams, - envVars, - contextVariables, - timeoutMs: timeout, - requestId, - ownerKey: `user:${auth.userId}`, - ownerWeight: 1, - }) + const isolatedResult = await executeInIsolatedVM( + { + code: codeToExecute, + params: executionParams, + envVars, + contextVariables, + timeoutMs: timeout, + requestId, + ownerKey: `user:${auth.userId}`, + ownerWeight: 1, + }, + { brokers: createFunctionRuntimeBrokers(routeContext) } + ) const executionTime = Date.now() - startTime @@ -1255,7 +1414,7 @@ export const POST = withRouteHandler(async (req: NextRequest) => { errorType: enhancedError.name, }) - return NextResponse.json( + return functionJsonResponse( { success: false, error: userFriendlyErrorMessage, @@ -1272,6 +1431,7 @@ export const POST = withRouteHandler(async (req: NextRequest) => { stack: enhancedError.stack, }, }, + routeContext, { status: isSystemError ? 500 : 422 } ) } @@ -1281,12 +1441,51 @@ export const POST = withRouteHandler(async (req: NextRequest) => { executionTime, }) - return NextResponse.json({ - success: true, - output: { result: isolatedResult.result, stdout: cleanStdout(stdout), executionTime }, - }) + return functionJsonResponse( + { + success: true, + output: { result: isolatedResult.result, stdout: cleanStdout(stdout), executionTime }, + }, + routeContext + ) } catch (error: any) { const executionTime = Date.now() - startTime + if (isExecutionResourceLimitError(error)) { + logger.warn(`[${requestId}] Function execution exceeded resource limits`, { + resource: error.resource, + attemptedBytes: error.attemptedBytes, + limitBytes: error.limitBytes, + executionTime, + }) + if (routeContext) { + return functionJsonResponse( + { + success: false, + error: error.message, + output: { + result: null, + stdout: cleanStdout(stdout), + executionTime, + }, + }, + routeContext, + { status: error.statusCode } + ) + } + return NextResponse.json( + { + success: false, + error: error.message, + output: { + result: null, + stdout: cleanStdout(stdout), + executionTime, + }, + }, + { status: error.statusCode } + ) + } + logger.error(`[${requestId}] Function execution failed`, { error: error.message || 'Unknown error', stack: error.stack, @@ -1328,6 +1527,10 @@ export const POST = withRouteHandler(async (req: NextRequest) => { }, } + if (routeContext) { + return functionJsonResponse(errorResponse, routeContext, { status: 500 }) + } + return NextResponse.json(errorResponse, { status: 500 }) } }) diff --git a/apps/sim/app/api/mothership/chats/[chatId]/route.test.ts b/apps/sim/app/api/mothership/chats/[chatId]/route.test.ts new file mode 100644 index 00000000000..ad0efd9a946 --- /dev/null +++ b/apps/sim/app/api/mothership/chats/[chatId]/route.test.ts @@ -0,0 +1,245 @@ +/** + * @vitest-environment node + */ +import { copilotHttpMock, copilotHttpMockFns } from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockGetAccessibleCopilotChat, + mockReconcileChatStreamMarkers, + mockReadEvents, + mockReadFilePreviewSessions, + mockGetLatestRunForStream, +} = vi.hoisted(() => ({ + mockGetAccessibleCopilotChat: vi.fn(), + mockReconcileChatStreamMarkers: vi.fn(), + mockReadEvents: vi.fn(), + mockReadFilePreviewSessions: vi.fn(), + mockGetLatestRunForStream: vi.fn(), +})) + +vi.mock('@sim/db', () => ({ db: {} })) + +vi.mock('@sim/db/schema', () => ({ + copilotChats: { + id: 'copilotChats.id', + userId: 'copilotChats.userId', + type: 'copilotChats.type', + updatedAt: 'copilotChats.updatedAt', + lastSeenAt: 'copilotChats.lastSeenAt', + }, +})) + +vi.mock('drizzle-orm', () => ({ + and: vi.fn((...conditions: unknown[]) => ({ type: 'and', conditions })), + eq: vi.fn((field: unknown, value: unknown) => ({ type: 'eq', field, value })), + sql: Object.assign( + vi.fn((strings: TemplateStringsArray, ...values: unknown[]) => ({ + type: 'sql', + strings, + values, + })), + { raw: vi.fn() } + ), +})) + +vi.mock('@/lib/copilot/request/http', () => copilotHttpMock) + +vi.mock('@/lib/copilot/chat/lifecycle', () => ({ + getAccessibleCopilotChat: mockGetAccessibleCopilotChat, +})) + +vi.mock('@/lib/copilot/chat/stream-liveness', () => ({ + reconcileChatStreamMarkers: mockReconcileChatStreamMarkers, +})) + +vi.mock('@/lib/copilot/request/session/buffer', () => ({ + readEvents: mockReadEvents, +})) + +vi.mock('@/lib/copilot/request/session/file-preview-session', () => ({ + readFilePreviewSessions: mockReadFilePreviewSessions, +})) + +vi.mock('@/lib/copilot/async-runs/repository', () => ({ + getLatestRunForStream: mockGetLatestRunForStream, +})) + +vi.mock('@/lib/copilot/request/session/types', () => ({ + toStreamBatchEvent: (e: unknown) => e, +})) + +vi.mock('@/lib/copilot/chat/effective-transcript', () => ({ + buildEffectiveChatTranscript: ({ messages }: { messages: unknown[] }) => messages, +})) + +vi.mock('@/lib/copilot/chat/persisted-message', () => ({ + normalizeMessage: (m: unknown) => m, +})) + +vi.mock('@/lib/copilot/tasks', () => ({ + taskPubSub: { publishStatusChanged: vi.fn() }, +})) + +vi.mock('@/lib/posthog/server', () => ({ + captureServerEvent: vi.fn(), +})) + +import { GET } from '@/app/api/mothership/chats/[chatId]/route' + +function makeContext(chatId: string) { + return { params: Promise.resolve({ chatId }) } +} + +function createRequest(chatId: string) { + return new NextRequest(`http://localhost:3000/api/mothership/chats/${chatId}`, { + method: 'GET', + }) +} + +describe('GET /api/mothership/chats/[chatId]', () => { + beforeEach(() => { + vi.clearAllMocks() + copilotHttpMockFns.mockAuthenticateCopilotRequestSessionOnly.mockResolvedValue({ + userId: 'user-1', + isAuthenticated: true, + }) + mockReconcileChatStreamMarkers.mockImplementation( + async (candidates: Array<{ chatId: string; streamId: string | null }>) => + new Map( + candidates.map((candidate) => [ + candidate.chatId, + { + chatId: candidate.chatId, + streamId: candidate.streamId, + status: candidate.streamId ? 'active' : 'inactive', + }, + ]) + ) + ) + mockReadEvents.mockResolvedValue([]) + mockReadFilePreviewSessions.mockResolvedValue([]) + mockGetLatestRunForStream.mockResolvedValue(null) + }) + + it('clears activeStreamId when the redis lock has expired (stuck-yellow bug)', async () => { + mockGetAccessibleCopilotChat.mockResolvedValueOnce({ + id: 'chat-stuck', + type: 'mothership', + title: 'Stuck', + messages: [], + resources: [], + conversationId: 'stream-orphaned', + createdAt: new Date('2026-05-11T12:00:00Z'), + updatedAt: new Date('2026-05-11T12:00:00Z'), + }) + mockReconcileChatStreamMarkers.mockResolvedValueOnce( + new Map([['chat-stuck', { chatId: 'chat-stuck', streamId: null, status: 'inactive' }]]) + ) + + const response = await GET(createRequest('chat-stuck'), makeContext('chat-stuck')) + expect(response.status).toBe(200) + const body = await response.json() + + expect(mockReconcileChatStreamMarkers).toHaveBeenCalledWith( + [{ chatId: 'chat-stuck', streamId: 'stream-orphaned' }], + { repairVerifiedStaleMarkers: true } + ) + expect(body.success).toBe(true) + expect(body.chat.activeStreamId).toBeNull() + expect(body.chat.streamSnapshot).toBeUndefined() + expect(mockReadEvents).not.toHaveBeenCalled() + }) + + it('returns the live activeStreamId when redis confirms the lock', async () => { + mockGetAccessibleCopilotChat.mockResolvedValueOnce({ + id: 'chat-live', + type: 'mothership', + title: 'Live', + messages: [], + resources: [], + conversationId: 'stream-live', + createdAt: new Date('2026-05-11T12:00:00Z'), + updatedAt: new Date('2026-05-11T12:00:00Z'), + }) + mockGetLatestRunForStream.mockResolvedValueOnce({ status: 'active' }) + + const response = await GET(createRequest('chat-live'), makeContext('chat-live')) + expect(response.status).toBe(200) + const body = await response.json() + + expect(body.chat.activeStreamId).toBe('stream-live') + expect(mockReadEvents).toHaveBeenCalledWith('stream-live', '0') + expect(body.chat.streamSnapshot).toBeDefined() + expect(body.chat.streamSnapshot.status).toBe('active') + }) + + it('uses the Redis lock owner when it differs from a stale persisted streamId', async () => { + mockGetAccessibleCopilotChat.mockResolvedValueOnce({ + id: 'chat-mismatch', + type: 'mothership', + title: 'Mismatch', + messages: [], + resources: [], + conversationId: 'stream-stale', + createdAt: new Date('2026-05-11T12:00:00Z'), + updatedAt: new Date('2026-05-11T12:00:00Z'), + }) + mockReconcileChatStreamMarkers.mockResolvedValueOnce( + new Map([ + ['chat-mismatch', { chatId: 'chat-mismatch', streamId: 'stream-live', status: 'active' }], + ]) + ) + + const response = await GET(createRequest('chat-mismatch'), makeContext('chat-mismatch')) + expect(response.status).toBe(200) + const body = await response.json() + + expect(body.chat.activeStreamId).toBe('stream-live') + expect(mockReadEvents).toHaveBeenCalledWith('stream-live', '0') + }) + + it('returns null when the persisted stream marker is already null', async () => { + mockGetAccessibleCopilotChat.mockResolvedValueOnce({ + id: 'chat-idle', + type: 'mothership', + title: 'Idle', + messages: [], + resources: [], + conversationId: null, + createdAt: new Date('2026-05-11T12:00:00Z'), + updatedAt: new Date('2026-05-11T12:00:00Z'), + }) + + const response = await GET(createRequest('chat-idle'), makeContext('chat-idle')) + expect(response.status).toBe(200) + + expect(mockReconcileChatStreamMarkers).toHaveBeenCalledWith( + [{ chatId: 'chat-idle', streamId: null }], + { repairVerifiedStaleMarkers: true } + ) + const body = await response.json() + expect(body.chat.activeStreamId).toBeNull() + }) + + it('returns 404 when the chat does not exist', async () => { + mockGetAccessibleCopilotChat.mockResolvedValueOnce(null) + + const response = await GET(createRequest('chat-missing'), makeContext('chat-missing')) + expect(response.status).toBe(404) + expect(mockReconcileChatStreamMarkers).not.toHaveBeenCalled() + }) + + it('returns 401 when unauthenticated', async () => { + copilotHttpMockFns.mockAuthenticateCopilotRequestSessionOnly.mockResolvedValueOnce({ + userId: null, + isAuthenticated: false, + }) + + const response = await GET(createRequest('chat-x'), makeContext('chat-x')) + expect(response.status).toBe(401) + expect(mockGetAccessibleCopilotChat).not.toHaveBeenCalled() + expect(mockReconcileChatStreamMarkers).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/mothership/chats/[chatId]/route.ts b/apps/sim/app/api/mothership/chats/[chatId]/route.ts index 62b7fd46181..b3a86bc8a2e 100644 --- a/apps/sim/app/api/mothership/chats/[chatId]/route.ts +++ b/apps/sim/app/api/mothership/chats/[chatId]/route.ts @@ -14,6 +14,7 @@ import { getLatestRunForStream } from '@/lib/copilot/async-runs/repository' import { buildEffectiveChatTranscript } from '@/lib/copilot/chat/effective-transcript' import { getAccessibleCopilotChat } from '@/lib/copilot/chat/lifecycle' import { normalizeMessage } from '@/lib/copilot/chat/persisted-message' +import { reconcileChatStreamMarkers } from '@/lib/copilot/chat/stream-liveness' import { authenticateCopilotRequestSessionOnly, createInternalServerErrorResponse, @@ -52,23 +53,29 @@ export const GET = withRouteHandler( status: string } | null = null - if (chat.conversationId) { + const reconciledMarkers = await reconcileChatStreamMarkers( + [{ chatId: chat.id, streamId: chat.conversationId }], + { repairVerifiedStaleMarkers: true } + ) + const liveStreamId = reconciledMarkers.get(chat.id)?.streamId ?? null + + if (liveStreamId) { try { const [events, previewSessions] = await Promise.all([ - readEvents(chat.conversationId, '0'), - readFilePreviewSessions(chat.conversationId).catch((error) => { + readEvents(liveStreamId, '0'), + readFilePreviewSessions(liveStreamId).catch((error) => { logger.warn('Failed to read preview sessions for mothership chat', { chatId, - conversationId: chat.conversationId, + streamId: liveStreamId, error: toError(error).message, }) return [] }), ]) - const run = await getLatestRunForStream(chat.conversationId, userId).catch((error) => { + const run = await getLatestRunForStream(liveStreamId, userId).catch((error) => { logger.warn('Failed to fetch latest run for mothership chat snapshot', { chatId, - conversationId: chat.conversationId, + streamId: liveStreamId, error: toError(error).message, }) return null @@ -87,7 +94,7 @@ export const GET = withRouteHandler( } catch (error) { logger.warn('Failed to read stream snapshot for mothership chat', { chatId, - conversationId: chat.conversationId, + streamId: liveStreamId, error: toError(error).message, }) } @@ -100,7 +107,7 @@ export const GET = withRouteHandler( : [] const effectiveMessages = buildEffectiveChatTranscript({ messages: normalizedMessages, - activeStreamId: chat.conversationId || null, + activeStreamId: liveStreamId, ...(streamSnapshot ? { streamSnapshot } : {}), }) @@ -110,7 +117,7 @@ export const GET = withRouteHandler( id: chat.id, title: chat.title, messages: effectiveMessages, - conversationId: chat.conversationId || null, + activeStreamId: liveStreamId, resources: Array.isArray(chat.resources) ? chat.resources : [], createdAt: chat.createdAt, updatedAt: chat.updatedAt, diff --git a/apps/sim/app/api/mothership/chats/route.test.ts b/apps/sim/app/api/mothership/chats/route.test.ts new file mode 100644 index 00000000000..5851d1b45df --- /dev/null +++ b/apps/sim/app/api/mothership/chats/route.test.ts @@ -0,0 +1,225 @@ +/** + * @vitest-environment node + */ +import { copilotHttpMock, copilotHttpMockFns, permissionsMock } from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockSelect, mockFrom, mockWhere, mockOrderBy, mockReconcileChatStreamMarkers } = vi.hoisted( + () => ({ + mockSelect: vi.fn(), + mockFrom: vi.fn(), + mockWhere: vi.fn(), + mockOrderBy: vi.fn(), + mockReconcileChatStreamMarkers: vi.fn(), + }) +) + +vi.mock('@sim/db', () => ({ + db: { + select: mockSelect, + }, +})) + +vi.mock('@sim/db/schema', () => ({ + copilotChats: { + id: 'copilotChats.id', + title: 'copilotChats.title', + userId: 'copilotChats.userId', + workspaceId: 'copilotChats.workspaceId', + type: 'copilotChats.type', + updatedAt: 'copilotChats.updatedAt', + conversationId: 'copilotChats.conversationId', + lastSeenAt: 'copilotChats.lastSeenAt', + }, +})) + +vi.mock('drizzle-orm', () => ({ + and: vi.fn((...conditions: unknown[]) => ({ type: 'and', conditions })), + desc: vi.fn((field: unknown) => ({ type: 'desc', field })), + eq: vi.fn((field: unknown, value: unknown) => ({ type: 'eq', field, value })), +})) + +vi.mock('@/lib/copilot/request/http', () => copilotHttpMock) +vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) + +vi.mock('@/lib/copilot/chat/stream-liveness', () => ({ + reconcileChatStreamMarkers: mockReconcileChatStreamMarkers, +})) + +vi.mock('@/lib/copilot/tasks', () => ({ + taskPubSub: { publishStatusChanged: vi.fn() }, +})) + +vi.mock('@/lib/posthog/server', () => ({ + captureServerEvent: vi.fn(), +})) + +import { GET } from '@/app/api/mothership/chats/route' + +function createRequest(workspaceId: string) { + return new NextRequest(`http://localhost:3000/api/mothership/chats?workspaceId=${workspaceId}`, { + method: 'GET', + }) +} + +describe('GET /api/mothership/chats', () => { + beforeEach(() => { + vi.clearAllMocks() + + copilotHttpMockFns.mockAuthenticateCopilotRequestSessionOnly.mockResolvedValue({ + userId: 'user-1', + isAuthenticated: true, + }) + + mockOrderBy.mockResolvedValue([]) + mockWhere.mockReturnValue({ orderBy: mockOrderBy }) + mockFrom.mockReturnValue({ where: mockWhere }) + mockSelect.mockReturnValue({ from: mockFrom }) + + mockReconcileChatStreamMarkers.mockImplementation( + async (candidates: Array<{ chatId: string; streamId: string | null }>) => + new Map( + candidates.map((candidate) => [ + candidate.chatId, + { + chatId: candidate.chatId, + streamId: candidate.streamId, + status: candidate.streamId ? 'active' : 'inactive', + }, + ]) + ) + ) + }) + + it('clears activeStreamId on chats whose redis lock has expired (stuck-yellow bug)', async () => { + const now = new Date('2026-05-11T12:00:00Z') + mockOrderBy.mockResolvedValueOnce([ + { + id: 'chat-stuck', + title: 'Stuck chat', + updatedAt: now, + activeStreamId: 'stream-orphaned', + lastSeenAt: null, + }, + { + id: 'chat-live', + title: 'Live chat', + updatedAt: now, + activeStreamId: 'stream-live', + lastSeenAt: null, + }, + { + id: 'chat-idle', + title: 'Idle chat', + updatedAt: now, + activeStreamId: null, + lastSeenAt: null, + }, + ]) + mockReconcileChatStreamMarkers.mockResolvedValueOnce( + new Map([ + ['chat-stuck', { chatId: 'chat-stuck', streamId: null, status: 'inactive' }], + ['chat-live', { chatId: 'chat-live', streamId: 'stream-live', status: 'active' }], + ['chat-idle', { chatId: 'chat-idle', streamId: null, status: 'inactive' }], + ]) + ) + + const response = await GET(createRequest('ws-1')) + expect(response.status).toBe(200) + const body = await response.json() + + expect(mockReconcileChatStreamMarkers).toHaveBeenCalledWith( + [ + { chatId: 'chat-stuck', streamId: 'stream-orphaned' }, + { chatId: 'chat-live', streamId: 'stream-live' }, + { chatId: 'chat-idle', streamId: null }, + ], + { repairVerifiedStaleMarkers: true } + ) + expect(body.success).toBe(true) + expect(body.data).toEqual([ + expect.objectContaining({ id: 'chat-stuck', activeStreamId: null }), + expect.objectContaining({ id: 'chat-live', activeStreamId: 'stream-live' }), + expect.objectContaining({ id: 'chat-idle', activeStreamId: null }), + ]) + }) + + it('preserves chats when no chat has a stream marker set', async () => { + const now = new Date('2026-05-11T12:00:00Z') + mockOrderBy.mockResolvedValueOnce([ + { id: 'chat-1', title: null, updatedAt: now, activeStreamId: null, lastSeenAt: null }, + { id: 'chat-2', title: null, updatedAt: now, activeStreamId: null, lastSeenAt: null }, + ]) + + const response = await GET(createRequest('ws-1')) + expect(response.status).toBe(200) + + expect(mockReconcileChatStreamMarkers).toHaveBeenCalledWith( + [ + { chatId: 'chat-1', streamId: null }, + { chatId: 'chat-2', streamId: null }, + ], + { repairVerifiedStaleMarkers: true } + ) + const body = await response.json() + expect(body.data).toEqual([ + expect.objectContaining({ id: 'chat-1', activeStreamId: null }), + expect.objectContaining({ id: 'chat-2', activeStreamId: null }), + ]) + }) + + it('leaves activeStreamId untouched when redis confirms every lock is live', async () => { + const now = new Date('2026-05-11T12:00:00Z') + mockOrderBy.mockResolvedValueOnce([ + { id: 'chat-a', title: null, updatedAt: now, activeStreamId: 'stream-a', lastSeenAt: null }, + { id: 'chat-b', title: null, updatedAt: now, activeStreamId: 'stream-b', lastSeenAt: null }, + ]) + + const response = await GET(createRequest('ws-1')) + const body = await response.json() + + expect(body.data).toEqual([ + expect.objectContaining({ id: 'chat-a', activeStreamId: 'stream-a' }), + expect.objectContaining({ id: 'chat-b', activeStreamId: 'stream-b' }), + ]) + }) + + it('uses Redis lock owner when it differs from a stale activeStreamId', async () => { + const now = new Date('2026-05-11T12:00:00Z') + mockOrderBy.mockResolvedValueOnce([ + { + id: 'chat-mismatch', + title: null, + updatedAt: now, + activeStreamId: 'stream-stale', + lastSeenAt: null, + }, + ]) + mockReconcileChatStreamMarkers.mockResolvedValueOnce( + new Map([ + ['chat-mismatch', { chatId: 'chat-mismatch', streamId: 'stream-live', status: 'active' }], + ]) + ) + + const response = await GET(createRequest('ws-1')) + expect(response.status).toBe(200) + const body = await response.json() + + expect(body.data).toEqual([ + expect.objectContaining({ id: 'chat-mismatch', activeStreamId: 'stream-live' }), + ]) + }) + + it('returns 401 when unauthenticated', async () => { + copilotHttpMockFns.mockAuthenticateCopilotRequestSessionOnly.mockResolvedValueOnce({ + userId: null, + isAuthenticated: false, + }) + + const response = await GET(createRequest('ws-1')) + expect(response.status).toBe(401) + expect(mockSelect).not.toHaveBeenCalled() + expect(mockReconcileChatStreamMarkers).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/mothership/chats/route.ts b/apps/sim/app/api/mothership/chats/route.ts index 3e114d16532..b0a068fabc1 100644 --- a/apps/sim/app/api/mothership/chats/route.ts +++ b/apps/sim/app/api/mothership/chats/route.ts @@ -8,6 +8,7 @@ import { listMothershipChatsContract, } from '@/lib/api/contracts/mothership-tasks' import { parseRequest } from '@/lib/api/server' +import { reconcileChatStreamMarkers } from '@/lib/copilot/chat/stream-liveness' import { authenticateCopilotRequestSessionOnly, createInternalServerErrorResponse, @@ -55,7 +56,16 @@ export const GET = withRouteHandler(async (request: NextRequest) => { ) .orderBy(desc(copilotChats.updatedAt)) - return NextResponse.json({ success: true, data: chats }) + const streamMarkers = await reconcileChatStreamMarkers( + chats.map((c) => ({ chatId: c.id, streamId: c.activeStreamId })), + { repairVerifiedStaleMarkers: true } + ) + const reconciled = chats.map((c) => { + const activeStreamId = streamMarkers.get(c.id)?.streamId ?? null + return activeStreamId === c.activeStreamId ? c : { ...c, activeStreamId } + }) + + return NextResponse.json({ success: true, data: reconciled }) } catch (error) { logger.error('Error fetching mothership chats:', error) return createInternalServerErrorResponse('Failed to fetch chats') diff --git a/apps/sim/app/api/mothership/execute/route.ts b/apps/sim/app/api/mothership/execute/route.ts index a85ca3c860f..d8a5e73f3d7 100644 --- a/apps/sim/app/api/mothership/execute/route.ts +++ b/apps/sim/app/api/mothership/execute/route.ts @@ -171,6 +171,7 @@ export const POST = withRouteHandler(async (req: NextRequest) => { return NextResponse.json({ content: result.content, model: 'mothership', + conversationId: effectiveChatId, tokens: result.usage ? { prompt: result.usage.prompt, diff --git a/apps/sim/app/api/resume/[workflowId]/[executionId]/[contextId]/route.ts b/apps/sim/app/api/resume/[workflowId]/[executionId]/[contextId]/route.ts index ff70c6f1898..47f2f381168 100644 --- a/apps/sim/app/api/resume/[workflowId]/[executionId]/[contextId]/route.ts +++ b/apps/sim/app/api/resume/[workflowId]/[executionId]/[contextId]/route.ts @@ -180,6 +180,10 @@ export const POST = withRouteHandler( timeoutMs: preprocessResult.executionTimeout?.sync, }, executionId: enqueueResult.resumeExecutionId, + workspaceId: workflow.workspaceId || undefined, + workflowId, + userId: enqueueResult.userId, + allowLargeValueWorkflowScope: true, executeFn: async ({ onStream, onBlockComplete, abortSignal }) => PauseResumeManager.startResumeExecution({ ...resumeArgs, diff --git a/apps/sim/app/api/tools/gmail/labels/route.ts b/apps/sim/app/api/tools/gmail/labels/route.ts index d675ea6e486..3b05cf12a9e 100644 --- a/apps/sim/app/api/tools/gmail/labels/route.ts +++ b/apps/sim/app/api/tools/gmail/labels/route.ts @@ -1,11 +1,8 @@ -import { db } from '@sim/db' -import { account } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { gmailLabelsSelectorContract } from '@/lib/api/contracts/selectors/google' import { parseRequest } from '@/lib/api/server' -import { getSession } from '@/lib/auth' +import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateAlphanumericId } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' @@ -13,7 +10,6 @@ import { getScopesForService } from '@/lib/oauth/utils' import { getServiceAccountToken, refreshAccessTokenIfNeeded, - resolveOAuthAccountId, ServiceAccountTokenError, } from '@/app/api/auth/oauth/utils' export const dynamic = 'force-dynamic' @@ -32,13 +28,6 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { - const session = await getSession() - - if (!session?.user?.id) { - logger.warn(`[${requestId}] Unauthenticated labels request rejected`) - return NextResponse.json({ error: 'User not authenticated' }, { status: 401 }) - } - const parsed = await parseRequest(gmailLabelsSelectorContract, request, {}) if (!parsed.success) return parsed.response const { credentialId, query } = parsed.data.query @@ -50,52 +39,26 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ error: credentialIdValidation.error }, { status: 400 }) } - const resolved = await resolveOAuthAccountId(credentialId) - if (!resolved) { - return NextResponse.json({ error: 'Credential not found' }, { status: 404 }) - } - - if (resolved.workspaceId) { - const { getUserEntityPermissions } = await import('@/lib/workspaces/permissions/utils') - const perm = await getUserEntityPermissions( - session.user.id, - 'workspace', - resolved.workspaceId - ) - if (perm === null) { - return NextResponse.json({ error: 'Forbidden' }, { status: 403 }) - } + const authz = await authorizeCredentialUse(request, { + credentialId, + requireWorkflowIdForInternal: false, + }) + if (!authz.ok || !authz.credentialOwnerUserId || !authz.resolvedCredentialId) { + return NextResponse.json({ error: authz.error || 'Unauthorized' }, { status: 403 }) } let accessToken: string | null = null - if (resolved.credentialType === 'service_account' && resolved.credentialId) { + if (authz.credentialType === 'service_account') { accessToken = await getServiceAccountToken( - resolved.credentialId, + authz.resolvedCredentialId, getScopesForService('gmail'), impersonateEmail ) } else { - const credentials = await db - .select() - .from(account) - .where(eq(account.id, resolved.accountId)) - .limit(1) - - if (!credentials.length) { - logger.warn(`[${requestId}] Credential not found`) - return NextResponse.json({ error: 'Credential not found' }, { status: 404 }) - } - - const accountRow = credentials[0] - - logger.info( - `[${requestId}] Using credential: ${accountRow.id}, provider: ${accountRow.providerId}` - ) - accessToken = await refreshAccessTokenIfNeeded( - resolved.accountId, - accountRow.userId, + credentialId, + authz.credentialOwnerUserId, requestId, getScopesForService('gmail') ) diff --git a/apps/sim/app/api/tools/onedrive/folders/route.ts b/apps/sim/app/api/tools/onedrive/folders/route.ts index 2538c6ae396..4c65c4190f6 100644 --- a/apps/sim/app/api/tools/onedrive/folders/route.ts +++ b/apps/sim/app/api/tools/onedrive/folders/route.ts @@ -1,15 +1,12 @@ -import { db } from '@sim/db' -import { account } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' -import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { onedriveFoldersQuerySchema } from '@/lib/api/contracts/selectors/microsoft' import { getValidationErrorMessage } from '@/lib/api/server' -import { getSession } from '@/lib/auth' +import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validateMicrosoftGraphId } from '@/lib/core/security/input-validation' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded, resolveOAuthAccountId } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' import type { MicrosoftGraphDriveItem } from '@/tools/onedrive/types' export const dynamic = 'force-dynamic' @@ -23,11 +20,6 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const requestId = generateId().slice(0, 8) try { - const session = await getSession() - if (!session?.user?.id) { - return NextResponse.json({ error: 'User not authenticated' }, { status: 401 }) - } - const { searchParams } = new URL(request.url) const validation = onedriveFoldersQuerySchema.safeParse({ credentialId: searchParams.get('credentialId') ?? '', @@ -51,37 +43,17 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ error: credentialIdValidation.error }, { status: 400 }) } - const resolved = await resolveOAuthAccountId(credentialId) - if (!resolved) { - return NextResponse.json({ error: 'Credential not found' }, { status: 404 }) - } - - if (resolved.workspaceId) { - const { getUserEntityPermissions } = await import('@/lib/workspaces/permissions/utils') - const perm = await getUserEntityPermissions( - session.user.id, - 'workspace', - resolved.workspaceId - ) - if (perm === null) { - return NextResponse.json({ error: 'Forbidden' }, { status: 403 }) - } - } - - const credentials = await db - .select() - .from(account) - .where(eq(account.id, resolved.accountId)) - .limit(1) - if (!credentials.length) { - return NextResponse.json({ error: 'Credential not found' }, { status: 404 }) + const authz = await authorizeCredentialUse(request, { + credentialId, + requireWorkflowIdForInternal: false, + }) + if (!authz.ok || !authz.credentialOwnerUserId || !authz.resolvedCredentialId) { + return NextResponse.json({ error: authz.error || 'Unauthorized' }, { status: 403 }) } - const accountRow = credentials[0] - const accessToken = await refreshAccessTokenIfNeeded( - resolved.accountId, - accountRow.userId, + credentialId, + authz.credentialOwnerUserId, requestId ) if (!accessToken) { diff --git a/apps/sim/app/api/tools/sftp/upload/route.ts b/apps/sim/app/api/tools/sftp/upload/route.ts index a0dbbbbbdb9..8acf93ca585 100644 --- a/apps/sim/app/api/tools/sftp/upload/route.ts +++ b/apps/sim/app/api/tools/sftp/upload/route.ts @@ -7,6 +7,7 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' import { createSftpConnection, getSftp, @@ -95,6 +96,13 @@ export const POST = withRouteHandler(async (request: NextRequest) => { for (const file of userFiles) { try { + const denied = await assertToolFileAccess( + file.key, + authResult.userId, + requestId, + logger + ) + if (denied) return denied logger.info( `[${requestId}] Downloading file for upload: ${file.name} (${file.size} bytes)` ) diff --git a/apps/sim/app/api/tools/sharepoint/upload/route.ts b/apps/sim/app/api/tools/sharepoint/upload/route.ts index 556de6d4225..2229d1ecc6a 100644 --- a/apps/sim/app/api/tools/sharepoint/upload/route.ts +++ b/apps/sim/app/api/tools/sharepoint/upload/route.ts @@ -9,6 +9,7 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' import type { MicrosoftGraphDriveItem } from '@/tools/onedrive/types' import type { SharepointSkippedFile, SharepointUploadError } from '@/tools/sharepoint/types' @@ -82,6 +83,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { const errors: SharepointUploadError[] = [] for (const userFile of userFiles) { + const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) + if (denied) return denied logger.info(`[${requestId}] Uploading file: ${userFile.name}`) const buffer = await downloadFileFromStorage(userFile, requestId, logger) diff --git a/apps/sim/app/api/tools/smtp/send/route.ts b/apps/sim/app/api/tools/smtp/send/route.ts index 127f8dca335..ea1f5e16d51 100644 --- a/apps/sim/app/api/tools/smtp/send/route.ts +++ b/apps/sim/app/api/tools/smtp/send/route.ts @@ -10,6 +10,7 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { processFilesToUserFiles } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -119,28 +120,25 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - const attachmentBuffers = await Promise.all( - attachments.map(async (file) => { - try { - logger.info( - `[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)` - ) - - const buffer = await downloadFileFromStorage(file, requestId, logger) - - return { - filename: file.name, - content: buffer, - contentType: file.type || 'application/octet-stream', - } - } catch (error) { - logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error) - throw new Error( - `Failed to download attachment "${file.name}": ${error instanceof Error ? error.message : 'Unknown error'}` - ) - } - }) - ) + const attachmentBuffers: { filename: string; content: Buffer; contentType: string }[] = [] + for (const file of attachments) { + const denied = await assertToolFileAccess(file.key, authResult.userId, requestId, logger) + if (denied) return denied + try { + logger.info(`[${requestId}] Downloading attachment: ${file.name} (${file.size} bytes)`) + const buffer = await downloadFileFromStorage(file, requestId, logger) + attachmentBuffers.push({ + filename: file.name, + content: buffer, + contentType: file.type || 'application/octet-stream', + }) + } catch (error) { + logger.error(`[${requestId}] Failed to download attachment ${file.name}:`, error) + throw new Error( + `Failed to download attachment "${file.name}": ${error instanceof Error ? error.message : 'Unknown error'}` + ) + } + } logger.info(`[${requestId}] Processed ${attachmentBuffers.length} attachment(s)`) mailOptions.attachments = attachmentBuffers diff --git a/apps/sim/app/api/tools/ssh/utils.ts b/apps/sim/app/api/tools/ssh/utils.ts index ed3dae88328..3d64440e22d 100644 --- a/apps/sim/app/api/tools/ssh/utils.ts +++ b/apps/sim/app/api/tools/ssh/utils.ts @@ -174,6 +174,8 @@ export async function createSSHConnection(config: SSHConnectionConfig): Promise< }) } +const MAX_OUTPUT_BYTES = 16 * 1024 * 1024 + /** * Execute a command on the SSH connection */ @@ -187,21 +189,45 @@ export function executeSSHCommand(client: Client, command: string): Promise { resolve({ - stdout: stdout.trim(), - stderr: stderr.trim(), - exitCode: code ?? 0, + stdout: stdoutTruncated + ? `${stdout.trim()}\n[output truncated: exceeded 16MB limit]` + : stdout.trim(), + stderr: stderrTruncated + ? `${stderr.trim()}\n[stderr truncated: exceeded 16MB limit]` + : stderr.trim(), + exitCode: code ?? -1, }) }) stream.on('data', (data: Buffer) => { - stdout += data.toString() + const remaining = MAX_OUTPUT_BYTES - stdoutBytes + if (remaining <= 0) { + stdoutTruncated = true + return + } + const chunk = data.subarray(0, remaining) + stdout += chunk.toString() + stdoutBytes += chunk.length + if (data.length > remaining) stdoutTruncated = true }) stream.stderr.on('data', (data: Buffer) => { - stderr += data.toString() + const remaining = MAX_OUTPUT_BYTES - stderrBytes + if (remaining <= 0) { + stderrTruncated = true + return + } + const chunk = data.subarray(0, remaining) + stderr += chunk.toString() + stderrBytes += chunk.length + if (data.length > remaining) stderrTruncated = true }) }) }) diff --git a/apps/sim/app/api/tools/wealthbox/items/route.ts b/apps/sim/app/api/tools/wealthbox/items/route.ts index 72fcd00b6f6..00ce1aab98a 100644 --- a/apps/sim/app/api/tools/wealthbox/items/route.ts +++ b/apps/sim/app/api/tools/wealthbox/items/route.ts @@ -1,15 +1,12 @@ -import { db } from '@sim/db' -import { account } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { eq } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { wealthboxItemsSelectorContract } from '@/lib/api/contracts/selectors/wealthbox' import { parseRequest } from '@/lib/api/server' -import { getSession } from '@/lib/auth' +import { authorizeCredentialUse } from '@/lib/auth/credential-access' import { validatePathSegment } from '@/lib/core/security/input-validation' import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' -import { refreshAccessTokenIfNeeded, resolveOAuthAccountId } from '@/app/api/auth/oauth/utils' +import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils' export const dynamic = 'force-dynamic' @@ -31,13 +28,6 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const requestId = generateRequestId() try { - const session = await getSession() - - if (!session?.user?.id) { - logger.warn(`[${requestId}] Unauthenticated request rejected`) - return NextResponse.json({ error: 'User not authenticated' }, { status: 401 }) - } - const parsed = await parseRequest(wealthboxItemsSelectorContract, request, {}) if (!parsed.success) return parsed.response const { credentialId, type } = parsed.data.query @@ -55,39 +45,17 @@ export const GET = withRouteHandler(async (request: NextRequest) => { return NextResponse.json({ error: credentialIdValidation.error }, { status: 400 }) } - const resolved = await resolveOAuthAccountId(credentialId) - if (!resolved) { - return NextResponse.json({ error: 'Credential not found' }, { status: 404 }) - } - - if (resolved.workspaceId) { - const { getUserEntityPermissions } = await import('@/lib/workspaces/permissions/utils') - const perm = await getUserEntityPermissions( - session.user.id, - 'workspace', - resolved.workspaceId - ) - if (perm === null) { - return NextResponse.json({ error: 'Forbidden' }, { status: 403 }) - } - } - - const credentials = await db - .select() - .from(account) - .where(eq(account.id, resolved.accountId)) - .limit(1) - - if (!credentials.length) { - logger.warn(`[${requestId}] Credential not found`, { credentialId }) - return NextResponse.json({ error: 'Credential not found' }, { status: 404 }) + const authz = await authorizeCredentialUse(request, { + credentialId, + requireWorkflowIdForInternal: false, + }) + if (!authz.ok || !authz.credentialOwnerUserId) { + return NextResponse.json({ error: authz.error || 'Unauthorized' }, { status: 403 }) } - const accountRow = credentials[0] - const accessToken = await refreshAccessTokenIfNeeded( - resolved.accountId, - accountRow.userId, + credentialId, + authz.credentialOwnerUserId, requestId ) diff --git a/apps/sim/app/api/tools/wordpress/upload/route.ts b/apps/sim/app/api/tools/wordpress/upload/route.ts index ed52dc65560..859aef52f54 100644 --- a/apps/sim/app/api/tools/wordpress/upload/route.ts +++ b/apps/sim/app/api/tools/wordpress/upload/route.ts @@ -11,7 +11,7 @@ import { processSingleFileToUserFile, } from '@/lib/uploads/utils/file-utils' import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' -import { verifyFileAccess } from '@/app/api/files/authorization' +import { assertToolFileAccess } from '@/app/api/files/authorization' export const dynamic = 'force-dynamic' @@ -78,22 +78,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) } - if (typeof userFile.key !== 'string' || userFile.key.length === 0) { - logger.warn(`[${requestId}] File access check rejected: missing key`) - return NextResponse.json({ success: false, error: 'File not found' }, { status: 404 }) - } - if (!authResult.userId) { - logger.warn(`[${requestId}] File access check requires userId but none available`) - return NextResponse.json({ success: false, error: 'File not found' }, { status: 404 }) - } - const hasAccess = await verifyFileAccess(userFile.key, authResult.userId) - if (!hasAccess) { - logger.warn(`[${requestId}] File access denied for user`, { - userId: authResult.userId, - key: userFile.key, - }) - return NextResponse.json({ success: false, error: 'File not found' }, { status: 404 }) - } + const denied = await assertToolFileAccess(userFile.key, authResult.userId, requestId, logger) + if (denied) return denied logger.info(`[${requestId}] Downloading file from storage`, { fileName: userFile.name, diff --git a/apps/sim/app/api/v1/admin/workflows/import/route.ts b/apps/sim/app/api/v1/admin/workflows/import/route.ts index cb38dbc5e82..5089e121157 100644 --- a/apps/sim/app/api/v1/admin/workflows/import/route.ts +++ b/apps/sim/app/api/v1/admin/workflows/import/route.ts @@ -34,6 +34,7 @@ import { } from '@/app/api/v1/admin/responses' import { extractWorkflowMetadata, + type VariableType, type WorkflowImportRequest, type WorkflowVariable, } from '@/app/api/v1/admin/types' @@ -118,14 +119,38 @@ export const POST = withRouteHandler( return internalErrorResponse(`Failed to save workflow state: ${saveResult.error}`) } - if (workflowData.variables && Array.isArray(workflowData.variables)) { + if ( + workflowData.variables && + typeof workflowData.variables === 'object' && + !Array.isArray(workflowData.variables) + ) { + const variablesRecord: Record = {} + const vars = workflowData.variables as Record< + string, + { id?: string; name: string; type?: VariableType; value: unknown } + > + Object.entries(vars).forEach(([key, v]) => { + const varId = v.id || key + variablesRecord[varId] = { + id: varId, + name: v.name, + type: v.type ?? 'string', + value: v.value, + } + }) + + await db + .update(workflow) + .set({ variables: variablesRecord, updatedAt: new Date() }) + .where(eq(workflow.id, workflowId)) + } else if (workflowData.variables && Array.isArray(workflowData.variables)) { const variablesRecord: Record = {} workflowData.variables.forEach((v) => { const varId = v.id || generateId() variablesRecord[varId] = { id: varId, name: v.name, - type: v.type || 'string', + type: (v.type as VariableType) ?? 'string', value: v.value, } }) diff --git a/apps/sim/app/api/workflows/[id]/execute/route.ts b/apps/sim/app/api/workflows/[id]/execute/route.ts index b0f0a0b1d4d..9d042cea756 100644 --- a/apps/sim/app/api/workflows/[id]/execute/route.ts +++ b/apps/sim/app/api/workflows/[id]/execute/route.ts @@ -36,6 +36,7 @@ import { registerManualExecutionAborter, unregisterManualExecutionAborter, } from '@/lib/execution/manual-cancellation' +import { compactBlockLogs, compactExecutionPayload } from '@/lib/execution/payloads/serializer' import { preprocessExecution } from '@/lib/execution/preprocessing' import { LoggingSession } from '@/lib/logs/execution/logging-session' import { @@ -65,7 +66,7 @@ import type { IterationContext, SerializableExecutionState, } from '@/executor/execution/types' -import type { NormalizedBlockOutput, StreamingExecution } from '@/executor/types' +import type { BlockLog, NormalizedBlockOutput, StreamingExecution } from '@/executor/types' import { getExecutionErrorStatus, hasExecutionResult } from '@/executor/utils/errors' import { Serializer } from '@/serializer' import { CORE_TRIGGER_TYPES, type CoreTriggerType } from '@/stores/logs/filters/types' @@ -75,6 +76,20 @@ const logger = createLogger('WorkflowExecuteAPI') export const runtime = 'nodejs' export const dynamic = 'force-dynamic' +async function compactRoutePayload( + value: T, + context: { + workspaceId?: string + workflowId?: string + executionId?: string + userId?: string + preserveUserFileBase64?: boolean + preserveRoot?: boolean + } +): Promise { + return compactExecutionPayload(value, { ...context, requireDurable: true }) +} + function resolveOutputIds( selectedOutputs: string[] | undefined, blocks: Record @@ -719,6 +734,14 @@ async function handleExecutePost( }) await handlePostExecutionPauseState({ result, workflowId, executionId, loggingSession }) + const compactResultOutput = await compactRoutePayload(result.output, { + workspaceId, + workflowId, + executionId, + userId: actorUserId, + preserveUserFileBase64: true, + preserveRoot: true, + }) if ( result.status === 'cancelled' && @@ -734,7 +757,7 @@ async function handleExecutePost( return NextResponse.json( { success: false, - output: result.output, + output: compactResultOutput, error: timeoutErrorMessage, metadata: result.metadata ? { @@ -751,21 +774,32 @@ async function handleExecutePost( const outputWithBase64 = includeFileBase64 ? ((await hydrateUserFilesWithBase64(result.output, { requestId, + workspaceId, + workflowId, executionId, + allowLargeValueWorkflowScope: Boolean(resolvedRunFromBlock?.sourceSnapshot), + userId: actorUserId, maxBytes: base64MaxBytes, })) as NormalizedBlockOutput) : result.output - const resultWithBase64 = { ...result, output: outputWithBase64 } - - if (auth.authType !== AuthType.INTERNAL_JWT && workflowHasResponseBlock(resultWithBase64)) { - return createHttpResponseFromBlock(resultWithBase64) + if (auth.authType !== AuthType.INTERNAL_JWT && workflowHasResponseBlock(result)) { + return createHttpResponseFromBlock({ ...result, output: outputWithBase64 }) } + const compactOutput = await compactRoutePayload(outputWithBase64, { + workspaceId, + workflowId, + executionId, + userId: actorUserId, + preserveUserFileBase64: true, + preserveRoot: true, + }) + const filteredResult = { success: result.success, executionId, - output: outputWithBase64, + output: compactOutput, error: result.error, metadata: result.metadata ? { @@ -784,11 +818,21 @@ async function handleExecutePost( const executionResult = hasExecutionResult(error) ? error.executionResult : undefined const status = getExecutionErrorStatus(error) + const compactErrorOutput = executionResult?.output + ? await compactRoutePayload(executionResult.output, { + workspaceId, + workflowId, + executionId, + userId: actorUserId, + preserveUserFileBase64: true, + preserveRoot: true, + }) + : undefined return NextResponse.json( { success: false, - output: executionResult?.output, + output: compactErrorOutput, error: executionResult?.error || errorMessage || 'Execution failed', metadata: executionResult?.metadata ? { @@ -838,6 +882,10 @@ async function handleExecutePost( timeoutMs: preprocessResult.executionTimeout?.sync, }, executionId, + workspaceId, + workflowId, + userId: actorUserId, + allowLargeValueWorkflowScope: Boolean(resolvedRunFromBlock?.sourceSnapshot), executeFn: async ({ onStream, onBlockComplete, abortSignal }) => executeWorkflow( streamWorkflow, @@ -856,6 +904,8 @@ async function handleExecutePost( base64MaxBytes, abortSignal, executionMode: 'stream', + stopAfterBlockId, + runFromBlock: resolvedRunFromBlock, }, executionId ), @@ -872,7 +922,12 @@ async function handleExecutePost( let isStreamClosed = false let isManualAbortRegistered = false - const eventWriter = createExecutionEventWriter(executionId) + const eventWriter = createExecutionEventWriter(executionId, { + workspaceId, + workflowId, + userId: actorUserId, + preserveUserFileBase64: includeFileBase64, + }) const metaInitialized = await initializeExecutionStreamMeta(executionId, { userId: actorUserId, workflowId, @@ -898,16 +953,18 @@ async function handleExecutePost( terminalStatus?: TerminalExecutionStreamStatus ) => { const isBuffered = event.type !== 'stream:chunk' && event.type !== 'stream:done' + let eventToSend = event if (isBuffered) { const entry = terminalStatus ? await eventWriter.writeTerminal(event, terminalStatus) : await eventWriter.write(event) - event.eventId = entry.eventId + eventToSend = entry.event + eventToSend.eventId = entry.eventId terminalEventPublished ||= Boolean(terminalStatus) } if (!isStreamClosed) { try { - controller.enqueue(encodeSSEEvent(event)) + controller.enqueue(encodeSSEEvent(eventToSend)) } catch { isStreamClosed = true } @@ -971,7 +1028,26 @@ async function handleExecutePost( iterationContext?: IterationContext, childWorkflowContext?: ChildWorkflowContext ) => { - const hasError = callbackData.output?.error + const compactCallbackData = { + ...callbackData, + input: await compactRoutePayload(callbackData.input, { + workspaceId, + workflowId, + executionId, + userId: actorUserId, + preserveUserFileBase64: includeFileBase64, + preserveRoot: true, + }), + output: await compactRoutePayload(callbackData.output, { + workspaceId, + workflowId, + executionId, + userId: actorUserId, + preserveUserFileBase64: includeFileBase64, + preserveRoot: true, + }), + } + const hasError = compactCallbackData.output?.error const childWorkflowData = childWorkflowContext ? { childWorkflowBlockId: childWorkflowContext.parentBlockId, @@ -988,7 +1064,7 @@ async function handleExecutePost( blockId, blockName, blockType, - error: callbackData.output.error, + error: compactCallbackData.output.error, }) await sendEvent({ type: 'block:error', @@ -999,12 +1075,12 @@ async function handleExecutePost( blockId, blockName, blockType, - input: callbackData.input, - error: callbackData.output.error, - durationMs: callbackData.executionTime || 0, - startedAt: callbackData.startedAt, - executionOrder: callbackData.executionOrder, - endedAt: callbackData.endedAt, + input: compactCallbackData.input, + error: compactCallbackData.output.error, + durationMs: compactCallbackData.executionTime || 0, + startedAt: compactCallbackData.startedAt, + executionOrder: compactCallbackData.executionOrder, + endedAt: compactCallbackData.endedAt, ...(iterationContext && { iterationCurrent: iterationContext.iterationCurrent, iterationTotal: iterationContext.iterationTotal, @@ -1033,12 +1109,12 @@ async function handleExecutePost( blockId, blockName, blockType, - input: callbackData.input, - output: callbackData.output, - durationMs: callbackData.executionTime || 0, - startedAt: callbackData.startedAt, - executionOrder: callbackData.executionOrder, - endedAt: callbackData.endedAt, + input: compactCallbackData.input, + output: compactCallbackData.output, + durationMs: compactCallbackData.executionTime || 0, + startedAt: compactCallbackData.startedAt, + executionOrder: compactCallbackData.executionOrder, + endedAt: compactCallbackData.endedAt, ...(iterationContext && { iterationCurrent: iterationContext.iterationCurrent, iterationTotal: iterationContext.iterationTotal, @@ -1172,6 +1248,20 @@ async function handleExecutePost( await handlePostExecutionPauseState({ result, workflowId, executionId, loggingSession }) + /** + * Compact block logs once and reuse across cancelled/timeout/paused/complete + * SSE events. Walks all block logs and durably serializes large values to + * object storage, so doing it twice would double the latency and storage + * load on the happy path. + */ + const compactedBlockLogs = await compactBlockLogs(result.logs, { + workspaceId, + workflowId, + executionId, + userId: actorUserId, + requireDurable: true, + }) + if (result.status === 'cancelled') { if (timeoutController.isTimedOut() && timeoutController.timeoutMs) { const timeoutErrorMessage = getTimeoutErrorMessage(null, timeoutController.timeoutMs) @@ -1191,7 +1281,7 @@ async function handleExecutePost( data: { error: timeoutErrorMessage, duration: result.metadata?.duration || 0, - finalBlockLogs: result.logs, + finalBlockLogs: compactedBlockLogs, }, }, 'error' @@ -1208,7 +1298,7 @@ async function handleExecutePost( workflowId, data: { duration: result.metadata?.duration || 0, - finalBlockLogs: result.logs, + finalBlockLogs: compactedBlockLogs, }, }, 'cancelled' @@ -1220,10 +1310,22 @@ async function handleExecutePost( const sseOutput = includeFileBase64 ? await hydrateUserFilesWithBase64(result.output, { requestId, + workspaceId, + workflowId, executionId, + allowLargeValueWorkflowScope: Boolean(resolvedRunFromBlock?.sourceSnapshot), + userId: actorUserId, maxBytes: base64MaxBytes, }) : result.output + const compactSseOutput = await compactRoutePayload(sseOutput, { + workspaceId, + workflowId, + executionId, + userId: actorUserId, + preserveUserFileBase64: true, + preserveRoot: true, + }) if (result.status === 'paused') { finalMetaStatus = 'complete' @@ -1234,11 +1336,11 @@ async function handleExecutePost( executionId, workflowId, data: { - output: sseOutput, + output: compactSseOutput, duration: result.metadata?.duration || 0, startTime: result.metadata?.startTime || startTime.toISOString(), endTime: result.metadata?.endTime || new Date().toISOString(), - finalBlockLogs: result.logs, + finalBlockLogs: compactedBlockLogs, }, }, 'complete' @@ -1253,11 +1355,11 @@ async function handleExecutePost( workflowId, data: { success: result.success, - output: sseOutput, + output: compactSseOutput, duration: result.metadata?.duration || 0, startTime: result.metadata?.startTime || startTime.toISOString(), endTime: result.metadata?.endTime || new Date().toISOString(), - finalBlockLogs: result.logs, + finalBlockLogs: compactedBlockLogs, }, }, 'complete' @@ -1274,6 +1376,22 @@ async function handleExecutePost( reqLogger.error(`SSE execution failed: ${errorMessage}`, { isTimeout }) const executionResult = hasExecutionResult(error) ? error.executionResult : undefined + let compactErrorLogs: BlockLog[] | undefined + try { + compactErrorLogs = executionResult?.logs + ? await compactBlockLogs(executionResult.logs, { + workspaceId, + workflowId, + executionId, + userId: actorUserId, + requireDurable: true, + }) + : undefined + } catch (compactionError) { + reqLogger.warn('Failed to compact SSE error logs, omitting oversized error details', { + error: toError(compactionError).message, + }) + } finalMetaStatus = 'error' await sendEvent( @@ -1285,7 +1403,7 @@ async function handleExecutePost( data: { error: executionResult?.error || errorMessage, duration: executionResult?.metadata?.duration || 0, - finalBlockLogs: executionResult?.logs, + finalBlockLogs: compactErrorLogs, }, }, 'error' diff --git a/apps/sim/app/api/workflows/[id]/executions/[executionId]/cancel/route.ts b/apps/sim/app/api/workflows/[id]/executions/[executionId]/cancel/route.ts index 02fab158465..92f32a26f7d 100644 --- a/apps/sim/app/api/workflows/[id]/executions/[executionId]/cancel/route.ts +++ b/apps/sim/app/api/workflows/[id]/executions/[executionId]/cancel/route.ts @@ -55,14 +55,19 @@ async function completePausedCancellationWithRetry( async function ensurePausedCancellationEventPublished( executionId: string, - workflowId: string + workflowId: string, + context: { workspaceId?: string; userId?: string } = {} ): Promise { const metaState = await readExecutionMetaState(executionId) if (metaState.status === 'found' && metaState.meta.status === 'cancelled') { return true } - const writer = createExecutionEventWriter(executionId) + const writer = createExecutionEventWriter(executionId, { + workspaceId: context.workspaceId, + workflowId, + userId: context.userId, + }) try { await writer.writeTerminal( { @@ -195,7 +200,11 @@ export const POST = withRouteHandler( if (pausedCancellationStarted) { pausedCancellationPublished = await ensurePausedCancellationEventPublished( executionId, - workflowId + workflowId, + { + workspaceId: workflowAuthorization.workflow?.workspaceId ?? undefined, + userId: auth.userId, + } ) pausedCancellationPublishFailed = !pausedCancellationPublished if (pausedCancellationPublished) { @@ -205,14 +214,22 @@ export const POST = withRouteHandler( if (pendingPausedCancellation === 'cancelled') { pausedCancellationPublished = await ensurePausedCancellationEventPublished( executionId, - workflowId + workflowId, + { + workspaceId: workflowAuthorization.workflow?.workspaceId ?? undefined, + userId: auth.userId, + } ) pausedCancellationPublishFailed = !pausedCancellationPublished pausedCancelled = pausedCancellationPublished } else if (pendingPausedCancellation === 'cancelling') { pausedCancellationPublished = await ensurePausedCancellationEventPublished( executionId, - workflowId + workflowId, + { + workspaceId: workflowAuthorization.workflow?.workspaceId ?? undefined, + userId: auth.userId, + } ) pausedCancellationPublishFailed = !pausedCancellationPublished if (pausedCancellationPublished) { diff --git a/apps/sim/app/api/workspaces/[id]/environment/route.ts b/apps/sim/app/api/workspaces/[id]/environment/route.ts index ec1e2b41120..23bd4bd18f1 100644 --- a/apps/sim/app/api/workspaces/[id]/environment/route.ts +++ b/apps/sim/app/api/workspaces/[id]/environment/route.ts @@ -3,7 +3,7 @@ import { db } from '@sim/db' import { workspaceEnvironment } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' -import { eq } from 'drizzle-orm' +import { eq, sql } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { removeWorkspaceEnvironmentContract, @@ -96,16 +96,6 @@ export const PUT = withRouteHandler( if (!parsed.success) return parsed.response const { variables } = parsed.data.body - // Read existing encrypted ws vars - const existingRows = await db - .select() - .from(workspaceEnvironment) - .where(eq(workspaceEnvironment.workspaceId, workspaceId)) - .limit(1) - - const existingEncrypted: Record = (existingRows[0]?.variables as any) || {} - - // Encrypt incoming const encryptedIncoming = await Promise.all( Object.entries(variables).map(async ([key, value]) => { const { encrypted } = await encryptSecret(value) @@ -113,22 +103,37 @@ export const PUT = withRouteHandler( }) ).then((entries) => Object.fromEntries(entries)) - const merged = { ...existingEncrypted, ...encryptedIncoming } - - // Upsert by unique workspace_id - await db - .insert(workspaceEnvironment) - .values({ - id: generateId(), - workspaceId, - variables: merged, - createdAt: new Date(), - updatedAt: new Date(), - }) - .onConflictDoUpdate({ - target: [workspaceEnvironment.workspaceId], - set: { variables: merged, updatedAt: new Date() }, - }) + const { existingEncrypted, merged } = await db.transaction(async (tx) => { + await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${workspaceId}))`) + + const [existingRow] = await tx + .select() + .from(workspaceEnvironment) + .where(eq(workspaceEnvironment.workspaceId, workspaceId)) + .limit(1) + + const existing = ((existingRow?.variables as Record) ?? {}) as Record< + string, + string + > + const mergedVars = { ...existing, ...encryptedIncoming } + + await tx + .insert(workspaceEnvironment) + .values({ + id: generateId(), + workspaceId, + variables: mergedVars, + createdAt: new Date(), + updatedAt: new Date(), + }) + .onConflictDoUpdate({ + target: [workspaceEnvironment.workspaceId], + set: { variables: mergedVars, updatedAt: new Date() }, + }) + + return { existingEncrypted: existing, merged: mergedVars } + }) const newKeys = Object.keys(variables).filter((k) => !(k in existingEncrypted)) await createWorkspaceEnvCredentials({ workspaceId, newKeys, actingUserId: userId }) @@ -183,39 +188,41 @@ export const DELETE = withRouteHandler( if (!parsed.success) return parsed.response const { keys } = parsed.data.body - const wsRows = await db - .select() - .from(workspaceEnvironment) - .where(eq(workspaceEnvironment.workspaceId, workspaceId)) - .limit(1) - - const current: Record = (wsRows[0]?.variables as any) || {} - let changed = false - for (const k of keys) { - if (k in current) { - delete current[k] - changed = true + const result = await db.transaction(async (tx) => { + await tx.execute(sql`SELECT pg_advisory_xact_lock(hashtext(${workspaceId}))`) + + const [existingRow] = await tx + .select() + .from(workspaceEnvironment) + .where(eq(workspaceEnvironment.workspaceId, workspaceId)) + .limit(1) + + if (!existingRow) return null + + const current: Record = + (existingRow.variables as Record) ?? {} + let modified = false + for (const k of keys) { + if (k in current) { + delete current[k] + modified = true + } } - } - if (!changed) { + if (!modified) return null + + await tx + .update(workspaceEnvironment) + .set({ variables: current, updatedAt: new Date() }) + .where(eq(workspaceEnvironment.workspaceId, workspaceId)) + + return { remainingKeysCount: Object.keys(current).length } + }) + + if (!result) { return NextResponse.json({ success: true }) } - await db - .insert(workspaceEnvironment) - .values({ - id: wsRows[0]?.id || generateId(), - workspaceId, - variables: current, - createdAt: wsRows[0]?.createdAt || new Date(), - updatedAt: new Date(), - }) - .onConflictDoUpdate({ - target: [workspaceEnvironment.workspaceId], - set: { variables: current, updatedAt: new Date() }, - }) - await deleteWorkspaceEnvCredentials({ workspaceId, removedKeys: keys }) recordAudit({ @@ -229,7 +236,7 @@ export const DELETE = withRouteHandler( description: `Removed ${keys.length} workspace environment variable(s)`, metadata: { removedKeys: keys, - remainingKeysCount: Object.keys(current).length, + remainingKeysCount: result.remainingKeysCount, }, request, }) diff --git a/apps/sim/app/chat/components/message/components/markdown-renderer.tsx b/apps/sim/app/chat/components/message/components/markdown-renderer.tsx index 32d49b167b2..84b60eea794 100644 --- a/apps/sim/app/chat/components/message/components/markdown-renderer.tsx +++ b/apps/sim/app/chat/components/message/components/markdown-renderer.tsx @@ -115,7 +115,7 @@ const COMPONENTS = { ), blockquote: ({ children }: React.HTMLAttributes) => ( -
    +
    {children}
    ), diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/preview-panel.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/preview-panel.tsx index 03ab43d4ade..53072bced76 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/preview-panel.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/preview-panel.tsx @@ -263,7 +263,7 @@ function CalloutBlock({ type, children }: { type: string; children?: React.React const config = CALLOUT_CONFIG[type] if (!config) { return ( -
    +
    {children}
    ) @@ -605,7 +605,7 @@ const STATIC_MARKDOWN_COMPONENTS = { return {children} } return ( -
    +
    {children}
    ) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.tsx index 62157c89eee..1537bef3925 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.tsx @@ -224,7 +224,7 @@ const MARKDOWN_COMPONENTS = { }, blockquote({ children }: { children?: React.ReactNode }) { return ( -
    +
    {children}
    ) diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts index d4a5e9628f1..680b96d8a17 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts @@ -3499,11 +3499,11 @@ export function useChat( processSSEStreamRef.current = processSSEStream const getActiveStreamIdForChat = useCallback( - async (chatId: string, signal?: AbortSignal): Promise => { + async ( + chatId: string, + signal?: AbortSignal + ): Promise<{ loaded: boolean; streamId: string | null }> => { const cached = queryClient.getQueryData(taskKeys.detail(chatId)) - if (cached?.activeStreamId) { - return cached.activeStreamId - } try { const fetchSignal = combineAbortSignals( @@ -3511,15 +3511,15 @@ export function useChat( createTimeoutSignal(CHAT_HISTORY_RECOVERY_TIMEOUT_MS) ) const history = await fetchChatHistory(chatId, fetchSignal) - if (signal?.aborted || fetchSignal?.aborted) return null + if (signal?.aborted || fetchSignal?.aborted) return { loaded: false, streamId: null } queryClient.setQueryData(taskKeys.detail(chatId), history) - return history.activeStreamId ?? null + return { loaded: true, streamId: history.activeStreamId ?? null } } catch (error) { logger.warn('Failed to load chat history while recovering stream', { chatId, error: toError(error).message, }) - return null + return { loaded: false, streamId: cached?.activeStreamId ?? null } } }, [queryClient] @@ -4032,12 +4032,12 @@ export function useChat( !recoveryController.signal.aborted const cached = queryClient.getQueryData(taskKeys.detail(chatId)) - let streamId = + const fallbackStreamId = streamIdRef.current ?? activeTurnRef.current?.userMessageId ?? cached?.activeStreamId - if (!streamId) { - streamId = - (await getActiveStreamIdForChat(chatId, recoveryController.signal)) ?? undefined - } + const loadedStream = await getActiveStreamIdForChat(chatId, recoveryController.signal) + const streamId = loadedStream.loaded + ? (loadedStream.streamId ?? undefined) + : fallbackStreamId if ( !isSameRecoverySubject() || streamGenRef.current !== observedGeneration || diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/note-block/note-block.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/note-block/note-block.tsx index 16a1291ca07..2c24056ca5c 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/note-block/note-block.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/note-block/note-block.tsx @@ -430,7 +430,7 @@ const NOTE_COMPONENTS = { {children} ), blockquote: ({ children }: { children?: React.ReactNode }) => ( -
    +
    {children}
    ), diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/input-mapping/input-mapping.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/input-mapping/input-mapping.tsx index a31c3b3c56e..9b13e9bebb4 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/input-mapping/input-mapping.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/input-mapping/input-mapping.tsx @@ -6,10 +6,11 @@ import { handleKeyboardActivation } from '@/lib/core/utils/keyboard' import { extractInputFieldsFromBlocks } from '@/lib/workflows/input-format' import { formatDisplayText } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/formatted-text' import { TagDropdown } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/tag-dropdown/tag-dropdown' +import { useDependsOnGate } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-depends-on-gate' import { useSubBlockInput } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-input' import { useSubBlockValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/hooks/use-sub-block-value' -import { resolvePreviewContextValue } from '@/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/utils' import { useAccessibleReferencePrefixes } from '@/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-accessible-reference-prefixes' +import type { SubBlockConfig } from '@/blocks/types' import { useWorkflowState } from '@/hooks/queries/workflows' /** @@ -35,7 +36,7 @@ interface InputMappingFieldProps { */ interface InputMappingProps { blockId: string - subBlockId: string + subBlock: SubBlockConfig isPreview?: boolean previewValue?: Record disabled?: boolean @@ -50,17 +51,20 @@ interface InputMappingProps { */ export function InputMapping({ blockId, - subBlockId, + subBlock, isPreview = false, previewValue, disabled = false, previewContextValues, }: InputMappingProps) { + const subBlockId = subBlock.id const [mapping, setMapping] = useSubBlockValue(blockId, subBlockId) - const [storeWorkflowId] = useSubBlockValue(blockId, 'workflowId') - const selectedWorkflowId = previewContextValues - ? resolvePreviewContextValue(previewContextValues.workflowId) - : storeWorkflowId + const { dependencyValues } = useDependsOnGate(blockId, subBlock, { + disabled, + isPreview, + previewContextValues, + }) + const selectedWorkflowId = dependencyValues.workflowId const inputController = useSubBlockInput({ blockId, diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx index f54cf6c42d3..0ffba587d18 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/sub-block.tsx @@ -1040,7 +1040,7 @@ function SubBlockComponent({ return ( @@ -149,13 +153,12 @@ export function SubflowEditor({ type='text' value={inputValue} onChange={handleSubflowIterationsChange} - onBlur={handleSubflowIterationsSave} - onKeyDown={(e) => e.key === 'Enter' && handleSubflowIterationsSave()} + onBlur={handleSubflowIterationsBlur} disabled={!userCanEdit} className='mb-1' />
    - Enter a number between 1 and {subflowConfig.maxIterations} + Enter a whole number greater than 0.
    ) : ( @@ -197,6 +200,33 @@ export function SubflowEditor({ )} + + {currentBlock.type === 'parallel' && ( +
    + + +
    + Run 1 to 20 parallel branches at a time. +
    +
    + )} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/hooks/use-subflow-editor.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/hooks/use-subflow-editor.ts index 08428f5d17c..915e7fb77dd 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/hooks/use-subflow-editor.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/hooks/use-subflow-editor.ts @@ -29,7 +29,6 @@ const SUBFLOW_CONFIG = { }, typeKey: 'loopType' as const, storeKey: 'loops' as const, - maxIterations: 1000, configKeys: { iterations: 'iterations' as const, items: 'forEachItems' as const, @@ -40,7 +39,6 @@ const SUBFLOW_CONFIG = { typeLabels: { count: 'Parallel Count', collection: 'Parallel Each' }, typeKey: 'parallelType' as const, storeKey: 'parallels' as const, - maxIterations: 20, configKeys: { iterations: 'count' as const, items: 'distribution' as const, @@ -61,9 +59,17 @@ export function useSubflowEditor(currentBlock: BlockState | null, currentBlockId const textareaRef = useRef(null) const editorContainerRef = useRef(null) - const [tempInputValue, setTempInputValue] = useState(null) const [showTagDropdown, setShowTagDropdown] = useState(false) const [cursorPosition, setCursorPosition] = useState(0) + /** + * In-flight string buffers for the numeric inputs. These let the user + * temporarily clear or mid-type the field (e.g. backspace to empty before + * typing a new value) without React snapping the value back from the store. + * Persistence still happens on every keystroke that parses to a number; + * the buffer is cleared on blur so the input rebinds to the store value. + */ + const [iterationsBuffer, setIterationsBuffer] = useState(null) + const [batchSizeBuffer, setBatchSizeBuffer] = useState(null) const isSubflow = currentBlock && (currentBlock.type === 'loop' || currentBlock.type === 'parallel') @@ -97,6 +103,7 @@ export function useSubflowEditor(currentBlock: BlockState | null, currentBlockId const { collaborativeUpdateLoopType, collaborativeUpdateParallelType, + collaborativeUpdateParallelBatchSize, collaborativeUpdateIterationCount, collaborativeUpdateIterationCollection, } = useCollaborativeWorkflow() @@ -218,47 +225,54 @@ export function useSubflowEditor(currentBlock: BlockState | null, currentBlockId ) /** - * Handle iterations input change + * Persist iterations on every keystroke that parses to a number. The + * visible string is buffered so transient states (empty, "0", partial typing) + * render correctly without snapping back to the persisted value. */ const handleSubflowIterationsChange = useCallback( (e: React.ChangeEvent) => { - if (!subflowConfig) return + if (!currentBlockId || !isSubflow || !subflowConfig || !currentBlock) return const sanitizedValue = e.target.value.replace(/[^0-9]/g, '') + setIterationsBuffer(sanitizedValue) const numValue = Number.parseInt(sanitizedValue) - - if (!Number.isNaN(numValue)) { - setTempInputValue(Math.min(subflowConfig.maxIterations, numValue).toString()) - } else { - setTempInputValue(sanitizedValue) - } + if (Number.isNaN(numValue)) return + collaborativeUpdateIterationCount( + currentBlockId, + currentBlock.type as 'loop' | 'parallel', + Math.max(1, numValue) + ) }, - [subflowConfig] + [currentBlockId, isSubflow, subflowConfig, currentBlock, collaborativeUpdateIterationCount] ) /** - * Save iterations value + * Clears the iterations buffer on blur so the field re-binds to the + * canonical store value (e.g. if the user left it empty, it snaps back + * to the last persisted count). */ - const handleSubflowIterationsSave = useCallback(() => { - if (!currentBlockId || !isSubflow || !subflowConfig || !currentBlock) return - const value = Number.parseInt(tempInputValue ?? '5') + const handleSubflowIterationsBlur = useCallback(() => { + setIterationsBuffer(null) + }, []) - if (!Number.isNaN(value)) { - const newValue = Math.min(subflowConfig.maxIterations, Math.max(1, value)) - collaborativeUpdateIterationCount( - currentBlockId, - currentBlock.type as 'loop' | 'parallel', - newValue - ) - } - setTempInputValue(null) - }, [ - tempInputValue, - currentBlockId, - isSubflow, - subflowConfig, - currentBlock, - collaborativeUpdateIterationCount, - ]) + /** + * Persist parallel batch size on every keystroke that parses to a number, + * clamped to 1..20. Buffered the same way as iterations. + */ + const handleParallelBatchSizeChange = useCallback( + (e: React.ChangeEvent) => { + if (!currentBlockId || currentBlock?.type !== 'parallel') return + const sanitizedValue = e.target.value.replace(/[^0-9]/g, '') + setBatchSizeBuffer(sanitizedValue) + const numValue = Number.parseInt(sanitizedValue) + if (Number.isNaN(numValue)) return + collaborativeUpdateParallelBatchSize(currentBlockId, Math.min(20, Math.max(1, numValue))) + }, + [currentBlockId, currentBlock, collaborativeUpdateParallelBatchSize] + ) + + const handleParallelBatchSizeBlur = useCallback(() => { + setBatchSizeBuffer(null) + }, []) /** * Handle editor value change (collection/condition) @@ -342,11 +356,16 @@ export function useSubflowEditor(currentBlock: BlockState | null, currentBlockId : '' const iterations = configIterations + const parallelBatchSize = + isSubflow && currentBlock?.type === 'parallel' + ? ((nodeConfig as any)?.batchSize ?? (blockData as any)?.batchSize ?? 20) + : 20 const collectionString = typeof configCollection === 'string' ? configCollection : JSON.stringify(configCollection) || '' const conditionString = typeof configCondition === 'string' ? configCondition : '' - const inputValue = tempInputValue ?? iterations.toString() + const inputValue = iterationsBuffer ?? iterations.toString() + const batchSizeValue = batchSizeBuffer ?? parallelBatchSize.toString() const editorValue = isConditionMode ? conditionString : collectionString // Type options for combobox @@ -366,6 +385,7 @@ export function useSubflowEditor(currentBlock: BlockState | null, currentBlockId isCountMode, isConditionMode, inputValue, + batchSizeValue, editorValue, typeOptions, showTagDropdown, @@ -376,7 +396,9 @@ export function useSubflowEditor(currentBlock: BlockState | null, currentBlockId // Handlers handleSubflowTypeChange, handleSubflowIterationsChange, - handleSubflowIterationsSave, + handleSubflowIterationsBlur, + handleParallelBatchSizeChange, + handleParallelBatchSizeBlur, handleSubflowEditorChange, handleSubflowTagSelect, highlightWithReferences, diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/terminal/components/output-panel/components/structured-output.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/terminal/components/output-panel/components/structured-output.tsx index 6cc83299279..3144de9d5e4 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/terminal/components/output-panel/components/structured-output.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/terminal/components/output-panel/components/structured-output.tsx @@ -14,6 +14,8 @@ import { import { List, type RowComponentProps, useListRef } from 'react-window' import { Badge, ChevronDown } from '@/components/emcn' import { cn } from '@/lib/core/utils/cn' +import { isUserFileDisplayMetadata } from '@/lib/core/utils/user-file' +import { isLargeValueRef, type LargeValueRef } from '@/lib/execution/payloads/large-value-ref' type ValueType = 'null' | 'undefined' | 'array' | 'string' | 'number' | 'boolean' | 'object' type BadgeVariant = 'green' | 'blue' | 'orange' | 'purple' | 'gray' | 'red' @@ -74,6 +76,19 @@ const STYLES = { } as const const EMPTY_MATCH_INDICES: number[] = [] +const USER_FILE_BASE64_PLACEHOLDER = '[TRUNCATED]' + +function formatLargeValueSize(bytes: number): string { + return `${(bytes / (1024 * 1024)).toFixed(1)} MB` +} + +function getLargeValueDisplayValue(ref: LargeValueRef): unknown { + return ref.preview ?? `[Large value: ${formatLargeValueSize(ref.size)}]` +} + +function getDisplayValue(value: unknown): unknown { + return isLargeValueRef(value) ? getLargeValueDisplayValue(value) : value +} function getTypeLabel(value: unknown): ValueType { if (value === null) return 'null' @@ -109,23 +124,39 @@ function extractErrorMessage(data: unknown): string { } function buildEntries(value: unknown, basePath: string): NodeEntry[] { - if (Array.isArray(value)) { - return value.map((item, i) => ({ key: String(i), value: item, path: `${basePath}[${i}]` })) + const displayValue = getDisplayValue(value) + + if (Array.isArray(displayValue)) { + return displayValue.map((item, i) => ({ + key: String(i), + value: item, + path: `${basePath}[${i}]`, + })) } - return Object.entries(value as Record).map(([k, v]) => ({ + const entries = Object.entries(displayValue as Record).map(([k, v]) => ({ key: k, value: v, path: `${basePath}.${k}`, })) + if (isUserFileDisplayMetadata(displayValue) && !('base64' in displayValue)) { + entries.push({ + key: 'base64', + value: USER_FILE_BASE64_PLACEHOLDER, + path: `${basePath}.base64`, + }) + } + return entries } function getCollapsedSummary(value: unknown): string | null { - if (Array.isArray(value)) { - const len = value.length + const displayValue = getDisplayValue(value) + + if (Array.isArray(displayValue)) { + const len = displayValue.length return `${len} item${len !== 1 ? 's' : ''}` } - if (typeof value === 'object' && value !== null) { - const count = Object.keys(value).length + if (typeof displayValue === 'object' && displayValue !== null) { + const count = buildEntries(displayValue, '').length return `${count} key${count !== 1 ? 's' : ''}` } return null @@ -133,10 +164,11 @@ function getCollapsedSummary(value: unknown): string | null { function computeInitialPaths(data: unknown, isError: boolean): Set { if (isError) return new Set(['root.error']) - if (!data || typeof data !== 'object') return new Set() - const entries = Array.isArray(data) - ? data.map((_, i) => `root[${i}]`) - : Object.keys(data).map((k) => `root.${k}`) + const displayData = getDisplayValue(data) + if (!displayData || typeof displayData !== 'object') return new Set() + const entries = Array.isArray(displayData) + ? displayData.map((_, i) => `root[${i}]`) + : Object.keys(displayData).map((k) => `root.${k}`) return new Set(entries) } @@ -184,13 +216,14 @@ function collectAllMatchPaths(data: unknown, query: string, basePath: string, de if (!query || depth > CONFIG.MAX_SEARCH_DEPTH) return [] const matches: string[] = [] + const displayData = getDisplayValue(data) - if (isPrimitive(data)) { - addPrimitiveMatches(data, `${basePath}.value`, query, matches) + if (isPrimitive(displayData)) { + addPrimitiveMatches(displayData, `${basePath}.value`, query, matches) return matches } - for (const entry of buildEntries(data, basePath)) { + for (const entry of buildEntries(displayData, basePath)) { if (isPrimitive(entry.value)) { addPrimitiveMatches(entry.value, entry.path, query, matches) } else { @@ -317,9 +350,10 @@ const StructuredNode = memo(function StructuredNode({ isError = false, }: StructuredNodeProps) { const searchContext = useContext(SearchContext) - const type = getTypeLabel(value) - const isPrimitiveValue = isPrimitive(value) - const isEmptyValue = !isPrimitiveValue && isEmpty(value) + const displayValue = getDisplayValue(value) + const type = getTypeLabel(displayValue) + const isPrimitiveValue = isPrimitive(displayValue) + const isEmptyValue = !isPrimitiveValue && isEmpty(displayValue) const isExpanded = expandedPaths.has(path) const handleToggle = useCallback(() => onToggle(path), [onToggle, path]) @@ -335,17 +369,17 @@ const StructuredNode = memo(function StructuredNode({ ) const childEntries = useMemo( - () => (isPrimitiveValue || isEmptyValue ? [] : buildEntries(value, path)), - [value, isPrimitiveValue, isEmptyValue, path] + () => (isPrimitiveValue || isEmptyValue ? [] : buildEntries(displayValue, path)), + [displayValue, isPrimitiveValue, isEmptyValue, path] ) const collapsedSummary = useMemo( - () => (isPrimitiveValue ? null : getCollapsedSummary(value)), - [value, isPrimitiveValue] + () => (isPrimitiveValue ? null : getCollapsedSummary(displayValue)), + [displayValue, isPrimitiveValue] ) const badgeVariant = isError ? 'red' : BADGE_VARIANTS[type] - const valueText = isPrimitiveValue ? formatPrimitive(value) : '' + const valueText = isPrimitiveValue ? formatPrimitive(displayValue) : '' const matchIndices = searchContext?.pathToMatchIndices.get(path) ?? EMPTY_MATCH_INDICES return ( @@ -472,16 +506,17 @@ function flattenTree( } function processNode(key: string, value: unknown, path: string, depth: number): void { - const valueType = getTypeLabel(value) - const isPrimitiveValue = isPrimitive(value) - const isEmptyValue = !isPrimitiveValue && isEmpty(value) + const displayValue = getDisplayValue(value) + const valueType = getTypeLabel(displayValue) + const isPrimitiveValue = isPrimitive(displayValue) + const isEmptyValue = !isPrimitiveValue && isEmpty(displayValue) const isExpanded = expandedPaths.has(path) - const collapsedSummary = isPrimitiveValue ? null : getCollapsedSummary(value) + const collapsedSummary = isPrimitiveValue ? null : getCollapsedSummary(displayValue) rows.push({ path, key, - value, + value: displayValue, depth, type: 'header', valueType, @@ -497,42 +532,43 @@ function flattenTree( rows.push({ path: `${path}.value`, key: '', - value, + value: displayValue, depth: depth + 1, type: 'value', valueType, isExpanded: false, isError: false, collapsedSummary: null, - displayText: formatPrimitive(value), + displayText: formatPrimitive(displayValue), matchIndices: pathToMatchIndices.get(path) ?? [], }) } else if (isEmptyValue) { rows.push({ path: `${path}.empty`, key: '', - value, + value: displayValue, depth: depth + 1, type: 'empty', valueType, isExpanded: false, isError: false, collapsedSummary: null, - displayText: Array.isArray(value) ? '[]' : '{}', + displayText: Array.isArray(displayValue) ? '[]' : '{}', matchIndices: [], }) } else { - for (const entry of buildEntries(value, path)) { + for (const entry of buildEntries(displayValue, path)) { processNode(entry.key, entry.value, entry.path, depth + 1) } } } } - if (isPrimitive(data)) { - processNode('value', data, 'root.value', 0) - } else if (data && typeof data === 'object') { - for (const entry of buildEntries(data, 'root')) { + const displayData = getDisplayValue(data) + if (isPrimitive(displayData)) { + processNode('value', displayData, 'root.value', 0) + } else if (displayData && typeof displayData === 'object') { + for (const entry of buildEntries(displayData, 'root')) { processNode(entry.key, entry.value, entry.path, 0) } } @@ -549,22 +585,24 @@ function countVisibleRows(data: unknown, expandedPaths: Set, isError: bo let count = 0 function countNode(value: unknown, path: string): void { + const displayValue = getDisplayValue(value) count++ if (!expandedPaths.has(path)) return - if (isPrimitive(value) || isEmpty(value)) { + if (isPrimitive(displayValue) || isEmpty(displayValue)) { count++ } else { - for (const entry of buildEntries(value, path)) { + for (const entry of buildEntries(displayValue, path)) { countNode(entry.value, entry.path) } } } - if (isPrimitive(data)) { - countNode(data, 'root.value') - } else if (data && typeof data === 'object') { - for (const entry of buildEntries(data, 'root')) { + const displayData = getDisplayValue(data) + if (isPrimitive(displayData)) { + countNode(displayData, 'root.value') + } else if (displayData && typeof displayData === 'object') { + for (const entry of buildEntries(displayData, 'root')) { countNode(entry.value, entry.path) } } @@ -782,8 +820,9 @@ export const StructuredOutput = memo(function StructuredOutput({ }, []) const rootEntries = useMemo(() => { - if (isPrimitive(data)) return [{ key: 'value', value: data, path: 'root.value' }] - return buildEntries(data, 'root') + const displayData = getDisplayValue(data) + if (isPrimitive(displayData)) return [{ key: 'value', value: displayData, path: 'root.value' }] + return buildEntries(displayData, 'root') }, [data]) const searchContextValue = useMemo(() => { diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.integration.test.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.integration.test.ts new file mode 100644 index 00000000000..f3b3d7cde6c --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.integration.test.ts @@ -0,0 +1,368 @@ +/** + * @vitest-environment node + * + * Integration tests that exercise `reconcileFinalBlockLogs` against the real + * `useTerminalConsoleStore` to validate end-to-end matching behavior. The + * sibling unit-test file mocks the store and only verifies call args, which + * cannot catch identity-mismatch regressions of the kind that produced the + * 34.57s wall-clock symptom. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.unmock('@/stores/terminal') +vi.unmock('@/stores/terminal/console/store') + +import { reconcileFinalBlockLogs } from '@/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils' +import { useExecutionStore } from '@/stores/execution' +import { useTerminalConsoleStore } from '@/stores/terminal/console/store' + +describe('reconcileFinalBlockLogs (real store)', () => { + beforeEach(() => { + useTerminalConsoleStore.setState({ + workflowEntries: {}, + entryIdsByBlockExecution: {}, + entryLocationById: {}, + isOpen: false, + _hasHydrated: true, + }) + vi.mocked(useExecutionStore.getState).mockReturnValue({ + getCurrentExecutionId: vi.fn(() => 'exec-1'), + } as any) + }) + + it('actually flips a child-workflow inner block from running to success', () => { + const store = useTerminalConsoleStore.getState() + store.addConsole({ + workflowId: 'wf-1', + blockId: 'workflow-1', + blockName: 'Workflow 1', + blockType: 'workflow', + executionId: 'exec-1', + executionOrder: 1, + isRunning: false, + success: true, + childWorkflowInstanceId: 'child-inst-1', + }) + store.addConsole({ + workflowId: 'wf-1', + blockId: 'set-projects', + blockName: 'setProjects', + blockType: 'variables', + executionId: 'exec-1', + executionOrder: 5, + isRunning: true, + childWorkflowBlockId: 'child-inst-1', + childWorkflowName: 'Workflow 1', + }) + + const startedAt = new Date().toISOString() + const endedAt = new Date(Date.now() + 27).toISOString() + + reconcileFinalBlockLogs(store.updateConsole, 'wf-1', 'exec-1', [ + { + blockId: 'workflow-1', + blockName: 'Workflow 1', + blockType: 'workflow', + startedAt, + endedAt, + durationMs: 100, + success: true, + executionOrder: 1, + childTraceSpans: [ + { + id: 'set-projects-span', + name: 'setProjects', + type: 'variables', + blockId: 'set-projects', + executionOrder: 5, + status: 'success', + duration: 27, + startTime: startedAt, + endTime: endedAt, + output: { value: [{ id: 'p1' }] }, + }, + ], + } as any, + ]) + + const innerEntry = useTerminalConsoleStore + .getState() + .getWorkflowEntries('wf-1') + .find((e) => e.blockId === 'set-projects') + + expect(innerEntry).toBeDefined() + expect(innerEntry?.isRunning).toBe(false) + expect(innerEntry?.success).toBe(true) + expect(innerEntry?.durationMs).toBe(27) + expect(innerEntry?.output).toEqual({ value: [{ id: 'p1' }] }) + }) + + it('targets the correct invocation when the same child nodeId runs twice', () => { + const store = useTerminalConsoleStore.getState() + store.addConsole({ + workflowId: 'wf-1', + blockId: 'workflow-1', + blockName: 'Workflow 1', + blockType: 'workflow', + executionId: 'exec-1', + executionOrder: 1, + isRunning: false, + success: true, + childWorkflowInstanceId: 'inst-A', + }) + store.addConsole({ + workflowId: 'wf-1', + blockId: 'workflow-1', + blockName: 'Workflow 1', + blockType: 'workflow', + executionId: 'exec-1', + executionOrder: 2, + isRunning: false, + success: true, + childWorkflowInstanceId: 'inst-B', + }) + store.addConsole({ + workflowId: 'wf-1', + blockId: 'fn-inner', + blockName: 'Inner', + blockType: 'function', + executionId: 'exec-1', + executionOrder: 3, + isRunning: true, + childWorkflowBlockId: 'inst-A', + }) + store.addConsole({ + workflowId: 'wf-1', + blockId: 'fn-inner', + blockName: 'Inner', + blockType: 'function', + executionId: 'exec-1', + executionOrder: 4, + isRunning: true, + childWorkflowBlockId: 'inst-B', + }) + + const startedAt = new Date().toISOString() + const endedAt = new Date(Date.now() + 5).toISOString() + const baseLog = { + blockName: 'Workflow 1', + blockType: 'workflow', + startedAt, + endedAt, + durationMs: 50, + success: true, + } + + reconcileFinalBlockLogs(store.updateConsole, 'wf-1', 'exec-1', [ + { + ...baseLog, + blockId: 'workflow-1', + executionOrder: 1, + childTraceSpans: [ + { + id: 'a', + name: 'Inner', + type: 'function', + blockId: 'fn-inner', + executionOrder: 3, + status: 'success', + duration: 5, + startTime: startedAt, + endTime: endedAt, + output: { result: 'A' }, + }, + ], + } as any, + { + ...baseLog, + blockId: 'workflow-1', + executionOrder: 2, + childTraceSpans: [ + { + id: 'b', + name: 'Inner', + type: 'function', + blockId: 'fn-inner', + executionOrder: 4, + status: 'success', + duration: 5, + startTime: startedAt, + endTime: endedAt, + output: { result: 'B' }, + }, + ], + } as any, + ]) + + const entries = useTerminalConsoleStore.getState().getWorkflowEntries('wf-1') + const a = entries.find((e) => e.blockId === 'fn-inner' && e.childWorkflowBlockId === 'inst-A') + const b = entries.find((e) => e.blockId === 'fn-inner' && e.childWorkflowBlockId === 'inst-B') + + expect(a?.isRunning).toBe(false) + expect(a?.output).toEqual({ result: 'A' }) + expect(b?.isRunning).toBe(false) + expect(b?.output).toEqual({ result: 'B' }) + }) + + it('propagates error state for spans with error status', () => { + const store = useTerminalConsoleStore.getState() + store.addConsole({ + workflowId: 'wf-1', + blockId: 'workflow-1', + blockName: 'Workflow 1', + blockType: 'workflow', + executionId: 'exec-1', + executionOrder: 1, + isRunning: false, + success: true, + childWorkflowInstanceId: 'inst-1', + }) + store.addConsole({ + workflowId: 'wf-1', + blockId: 'http-1', + blockName: 'API', + blockType: 'api', + executionId: 'exec-1', + executionOrder: 2, + isRunning: true, + childWorkflowBlockId: 'inst-1', + }) + + const startedAt = new Date().toISOString() + const endedAt = new Date(Date.now() + 30).toISOString() + + reconcileFinalBlockLogs(store.updateConsole, 'wf-1', 'exec-1', [ + { + blockId: 'workflow-1', + blockName: 'Workflow 1', + blockType: 'workflow', + startedAt, + endedAt, + durationMs: 100, + success: true, + executionOrder: 1, + childTraceSpans: [ + { + id: 'http-span', + name: 'API', + type: 'api', + blockId: 'http-1', + executionOrder: 2, + status: 'error', + duration: 30, + startTime: startedAt, + endTime: endedAt, + output: { error: 'Connection refused' }, + }, + ], + } as any, + ]) + + const entry = useTerminalConsoleStore + .getState() + .getWorkflowEntries('wf-1') + .find((e) => e.blockId === 'http-1') + + expect(entry?.isRunning).toBe(false) + expect(entry?.success).toBe(false) + expect(entry?.error).toBe('Connection refused') + }) + + it('matches the correct iteration row inside a child workflow loop', () => { + const store = useTerminalConsoleStore.getState() + store.addConsole({ + workflowId: 'wf-1', + blockId: 'workflow-1', + blockName: 'Workflow 1', + blockType: 'workflow', + executionId: 'exec-1', + executionOrder: 1, + isRunning: false, + success: true, + childWorkflowInstanceId: 'inst-1', + }) + store.addConsole({ + workflowId: 'wf-1', + blockId: 'fn-leaf', + blockName: 'Leaf', + blockType: 'function', + executionId: 'exec-1', + executionOrder: 2, + isRunning: false, + success: true, + iterationCurrent: 0, + iterationType: 'loop', + iterationContainerId: 'loop-1', + childWorkflowBlockId: 'inst-1', + output: { i: 0 }, + }) + store.addConsole({ + workflowId: 'wf-1', + blockId: 'fn-leaf', + blockName: 'Leaf', + blockType: 'function', + executionId: 'exec-1', + executionOrder: 3, + isRunning: true, + iterationCurrent: 1, + iterationType: 'loop', + iterationContainerId: 'loop-1', + childWorkflowBlockId: 'inst-1', + }) + + const startedAt = new Date().toISOString() + const endedAt = new Date(Date.now() + 12).toISOString() + + reconcileFinalBlockLogs(store.updateConsole, 'wf-1', 'exec-1', [ + { + blockId: 'workflow-1', + blockName: 'Workflow 1', + blockType: 'workflow', + startedAt, + endedAt, + durationMs: 100, + success: true, + executionOrder: 1, + childTraceSpans: [ + { + id: 'leaf-0', + name: 'Leaf', + type: 'function', + blockId: 'fn-leaf', + executionOrder: 2, + loopId: 'loop-1', + iterationIndex: 0, + status: 'success', + duration: 5, + startTime: startedAt, + endTime: endedAt, + output: { i: 0 }, + }, + { + id: 'leaf-1', + name: 'Leaf', + type: 'function', + blockId: 'fn-leaf', + executionOrder: 3, + loopId: 'loop-1', + iterationIndex: 1, + status: 'success', + duration: 12, + startTime: startedAt, + endTime: endedAt, + output: { i: 1 }, + }, + ], + } as any, + ]) + + const entries = useTerminalConsoleStore.getState().getWorkflowEntries('wf-1') + const iter0 = entries.find((e) => e.blockId === 'fn-leaf' && e.iterationCurrent === 0) + const iter1 = entries.find((e) => e.blockId === 'fn-leaf' && e.iterationCurrent === 1) + + expect(iter0?.isRunning).toBe(false) + expect(iter0?.output).toEqual({ i: 0 }) + expect(iter1?.isRunning).toBe(false) + expect(iter1?.output).toEqual({ i: 1 }) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.test.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.test.ts index d2c999beef0..e7272f901cf 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.test.ts @@ -427,7 +427,7 @@ describe('workflow-execution-utils', () => { executionId: 'exec-1', executionOrder: 3, isRunning: true, - childWorkflowBlockId: 'workflow-1', + childWorkflowBlockId: 'child-inst-1', childWorkflowName: 'Workflow 1', }) terminalConsoleMockFns.mockAddConsole({ @@ -489,7 +489,7 @@ describe('workflow-execution-utils', () => { success: true, isRunning: false, isCanceled: false, - childWorkflowBlockId: 'workflow-1', + childWorkflowBlockId: 'child-inst-1', }), 'exec-1', ]) @@ -501,7 +501,7 @@ describe('workflow-execution-utils', () => { error: 'Request failed', isRunning: false, isCanceled: false, - childWorkflowBlockId: 'workflow-1', + childWorkflowBlockId: 'child-inst-1', }), 'exec-1', ]) @@ -529,7 +529,7 @@ describe('workflow-execution-utils', () => { iterationCurrent: 0, iterationType: 'loop', iterationContainerId: 'loop-1', - childWorkflowBlockId: 'workflow-1', + childWorkflowBlockId: 'child-inst-1', }) terminalConsoleMockFns.mockAddConsole({ workflowId: 'wf-1', @@ -542,7 +542,7 @@ describe('workflow-execution-utils', () => { iterationCurrent: 1, iterationType: 'loop', iterationContainerId: 'loop-1', - childWorkflowBlockId: 'workflow-1', + childWorkflowBlockId: 'child-inst-1', }) const startedAt = new Date().toISOString() @@ -632,7 +632,7 @@ describe('workflow-execution-utils', () => { executionId: 'exec-1', executionOrder: 3, isRunning: false, - childWorkflowBlockId: 'workflow-1', + childWorkflowBlockId: 'child-inst-1', childWorkflowInstanceId: 'nested-inst-1', }) terminalConsoleMockFns.mockAddConsole({ @@ -643,7 +643,7 @@ describe('workflow-execution-utils', () => { executionId: 'exec-1', executionOrder: 1, isRunning: true, - childWorkflowBlockId: 'nested-workflow', + childWorkflowBlockId: 'nested-inst-1', }) const startedAt = new Date().toISOString() @@ -688,7 +688,7 @@ describe('workflow-execution-utils', () => { expect(updateConsole.mock.calls[1]).toEqual([ 'nested-api', expect.objectContaining({ - childWorkflowBlockId: 'nested-workflow', + childWorkflowBlockId: 'nested-inst-1', success: true, isRunning: false, isCanceled: false, @@ -697,6 +697,387 @@ describe('workflow-execution-utils', () => { ]) }) + it('rescues a child-workflow block whose block:completed SSE event was dropped', () => { + terminalConsoleMockFns.mockAddConsole({ + workflowId: 'wf-1', + blockId: 'workflow-1', + blockName: 'Workflow 1', + blockType: 'workflow', + executionId: 'exec-1', + executionOrder: 1, + success: true, + isRunning: false, + childWorkflowInstanceId: 'child-inst-1', + }) + terminalConsoleMockFns.mockAddConsole({ + workflowId: 'wf-1', + blockId: 'set-projects', + blockName: 'setProjects', + blockType: 'variables', + executionId: 'exec-1', + executionOrder: 5, + isRunning: true, + childWorkflowBlockId: 'child-inst-1', + childWorkflowName: 'Workflow 1', + }) + + const startedAt = new Date().toISOString() + const endedAt = new Date(Date.now() + 27).toISOString() + const updateConsole = vi.fn() + reconcileFinalBlockLogs(updateConsole, 'wf-1', 'exec-1', [ + makeLog({ + blockId: 'workflow-1', + blockType: 'workflow', + executionOrder: 1, + childTraceSpans: [ + { + id: 'set-projects-span', + name: 'setProjects', + type: 'variables', + blockId: 'set-projects', + executionOrder: 5, + status: 'success', + duration: 27, + startTime: startedAt, + endTime: endedAt, + output: { value: [{ id: 'p1' }, { id: 'p2' }] }, + }, + ], + }), + ]) + + expect(updateConsole).toHaveBeenCalledTimes(1) + expect(updateConsole.mock.calls[0]).toEqual([ + 'set-projects', + expect.objectContaining({ + executionOrder: 5, + childWorkflowBlockId: 'child-inst-1', + replaceOutput: { value: [{ id: 'p1' }, { id: 'p2' }] }, + success: true, + isRunning: false, + isCanceled: false, + durationMs: 27, + startedAt, + endedAt, + }), + 'exec-1', + ]) + }) + + it('matches per-invocation when the same child workflow nodeId runs twice', () => { + terminalConsoleMockFns.mockAddConsole({ + workflowId: 'wf-1', + blockId: 'workflow-1', + blockName: 'Workflow 1', + blockType: 'workflow', + executionId: 'exec-1', + executionOrder: 1, + success: true, + childWorkflowInstanceId: 'inst-A', + }) + terminalConsoleMockFns.mockAddConsole({ + workflowId: 'wf-1', + blockId: 'workflow-1', + blockName: 'Workflow 1', + blockType: 'workflow', + executionId: 'exec-1', + executionOrder: 2, + success: true, + childWorkflowInstanceId: 'inst-B', + }) + terminalConsoleMockFns.mockAddConsole({ + workflowId: 'wf-1', + blockId: 'fn-inner', + blockName: 'Inner', + blockType: 'function', + executionId: 'exec-1', + executionOrder: 3, + isRunning: true, + childWorkflowBlockId: 'inst-A', + }) + terminalConsoleMockFns.mockAddConsole({ + workflowId: 'wf-1', + blockId: 'fn-inner', + blockName: 'Inner', + blockType: 'function', + executionId: 'exec-1', + executionOrder: 4, + isRunning: true, + childWorkflowBlockId: 'inst-B', + }) + + const startedAt = new Date().toISOString() + const endedAt = new Date(Date.now() + 10).toISOString() + const updateConsole = vi.fn() + reconcileFinalBlockLogs(updateConsole, 'wf-1', 'exec-1', [ + makeLog({ + blockId: 'workflow-1', + blockType: 'workflow', + executionOrder: 1, + childTraceSpans: [ + { + id: 'a', + name: 'Inner', + type: 'function', + blockId: 'fn-inner', + executionOrder: 3, + status: 'success', + duration: 5, + startTime: startedAt, + endTime: endedAt, + output: { result: 'A' }, + }, + ], + }), + makeLog({ + blockId: 'workflow-1', + blockType: 'workflow', + executionOrder: 2, + childTraceSpans: [ + { + id: 'b', + name: 'Inner', + type: 'function', + blockId: 'fn-inner', + executionOrder: 4, + status: 'success', + duration: 5, + startTime: startedAt, + endTime: endedAt, + output: { result: 'B' }, + }, + ], + }), + ]) + + expect(updateConsole).toHaveBeenCalledTimes(2) + expect(updateConsole.mock.calls[0][1]).toMatchObject({ + executionOrder: 3, + childWorkflowBlockId: 'inst-A', + replaceOutput: { result: 'A' }, + }) + expect(updateConsole.mock.calls[1][1]).toMatchObject({ + executionOrder: 4, + childWorkflowBlockId: 'inst-B', + replaceOutput: { result: 'B' }, + }) + }) + + it('reconciles parallel-iteration spans inside a child workflow', () => { + terminalConsoleMockFns.mockAddConsole({ + workflowId: 'wf-1', + blockId: 'workflow-1', + blockType: 'workflow', + blockName: 'Workflow 1', + executionId: 'exec-1', + executionOrder: 1, + success: true, + childWorkflowInstanceId: 'inst-1', + }) + terminalConsoleMockFns.mockAddConsole({ + workflowId: 'wf-1', + blockId: 'fn-leaf', + blockType: 'function', + blockName: 'Leaf', + executionId: 'exec-1', + executionOrder: 2, + isRunning: true, + iterationCurrent: 0, + iterationType: 'parallel', + iterationContainerId: 'par-1', + childWorkflowBlockId: 'inst-1', + }) + + const startedAt = new Date().toISOString() + const endedAt = new Date(Date.now() + 8).toISOString() + const updateConsole = vi.fn() + reconcileFinalBlockLogs(updateConsole, 'wf-1', 'exec-1', [ + makeLog({ + blockId: 'workflow-1', + blockType: 'workflow', + executionOrder: 1, + childTraceSpans: [ + { + id: 'leaf-span', + name: 'Leaf', + type: 'function', + blockId: 'fn-leaf', + executionOrder: 2, + parallelId: 'par-1', + iterationIndex: 0, + status: 'success', + duration: 8, + startTime: startedAt, + endTime: endedAt, + output: { ok: true }, + }, + ], + }), + ]) + + expect(updateConsole).toHaveBeenCalledTimes(1) + expect(updateConsole.mock.calls[0][1]).toMatchObject({ + executionOrder: 2, + iterationCurrent: 0, + iterationType: 'parallel', + iterationContainerId: 'par-1', + childWorkflowBlockId: 'inst-1', + success: true, + }) + }) + + it('rescues only the iteration whose terminal SSE event was dropped', () => { + terminalConsoleMockFns.mockAddConsole({ + workflowId: 'wf-1', + blockId: 'workflow-1', + blockType: 'workflow', + blockName: 'Workflow 1', + executionId: 'exec-1', + executionOrder: 1, + success: true, + childWorkflowInstanceId: 'inst-1', + }) + terminalConsoleMockFns.mockAddConsole({ + workflowId: 'wf-1', + blockId: 'fn-leaf', + blockType: 'function', + blockName: 'Leaf', + executionId: 'exec-1', + executionOrder: 2, + isRunning: false, + success: true, + iterationCurrent: 0, + iterationType: 'loop', + iterationContainerId: 'loop-1', + childWorkflowBlockId: 'inst-1', + }) + terminalConsoleMockFns.mockAddConsole({ + workflowId: 'wf-1', + blockId: 'fn-leaf', + blockType: 'function', + blockName: 'Leaf', + executionId: 'exec-1', + executionOrder: 3, + isRunning: true, + iterationCurrent: 1, + iterationType: 'loop', + iterationContainerId: 'loop-1', + childWorkflowBlockId: 'inst-1', + }) + + const startedAt = new Date().toISOString() + const endedAt = new Date(Date.now() + 12).toISOString() + const updateConsole = vi.fn() + reconcileFinalBlockLogs(updateConsole, 'wf-1', 'exec-1', [ + makeLog({ + blockId: 'workflow-1', + blockType: 'workflow', + executionOrder: 1, + childTraceSpans: [ + { + id: 'leaf-0', + name: 'Leaf', + type: 'function', + blockId: 'fn-leaf', + executionOrder: 2, + loopId: 'loop-1', + iterationIndex: 0, + status: 'success', + duration: 5, + startTime: startedAt, + endTime: endedAt, + output: { i: 0 }, + }, + { + id: 'leaf-1', + name: 'Leaf', + type: 'function', + blockId: 'fn-leaf', + executionOrder: 3, + loopId: 'loop-1', + iterationIndex: 1, + status: 'success', + duration: 12, + startTime: startedAt, + endTime: endedAt, + output: { i: 1 }, + }, + ], + }), + ]) + + // updateConsole is called for both spans (idempotent re-application), but + // production matchesEntryForUpdate filters by the identity so only the + // still-running iteration is actually mutated. We assert the args carry + // distinct iteration identities so the store can target the right row. + expect(updateConsole.mock.calls[0][1]).toMatchObject({ + executionOrder: 2, + iterationCurrent: 0, + }) + expect(updateConsole.mock.calls[1][1]).toMatchObject({ + executionOrder: 3, + iterationCurrent: 1, + replaceOutput: { i: 1 }, + }) + }) + + it('propagates span error state when the block:error SSE was lost', () => { + terminalConsoleMockFns.mockAddConsole({ + workflowId: 'wf-1', + blockId: 'workflow-1', + blockType: 'workflow', + blockName: 'Workflow 1', + executionId: 'exec-1', + executionOrder: 1, + success: true, + childWorkflowInstanceId: 'inst-1', + }) + terminalConsoleMockFns.mockAddConsole({ + workflowId: 'wf-1', + blockId: 'http-1', + blockType: 'api', + blockName: 'API', + executionId: 'exec-1', + executionOrder: 2, + isRunning: true, + childWorkflowBlockId: 'inst-1', + }) + + const startedAt = new Date().toISOString() + const endedAt = new Date(Date.now() + 30).toISOString() + const updateConsole = vi.fn() + reconcileFinalBlockLogs(updateConsole, 'wf-1', 'exec-1', [ + makeLog({ + blockId: 'workflow-1', + blockType: 'workflow', + executionOrder: 1, + childTraceSpans: [ + { + id: 'http-span', + name: 'API', + type: 'api', + blockId: 'http-1', + executionOrder: 2, + status: 'error', + duration: 30, + startTime: startedAt, + endTime: endedAt, + output: { error: 'Connection refused' }, + }, + ], + }), + ]) + + expect(updateConsole).toHaveBeenCalledTimes(1) + expect(updateConsole.mock.calls[0][1]).toMatchObject({ + success: false, + error: 'Connection refused', + childWorkflowBlockId: 'inst-1', + isRunning: false, + isCanceled: false, + }) + }) + it('is a no-op when finalBlockLogs is empty or executionId is missing', () => { const updateConsole = vi.fn() reconcileFinalBlockLogs(updateConsole, 'wf-1', 'exec-1', []) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.ts index 008a1567cd9..4b9e726d089 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/utils/workflow-execution-utils.ts @@ -512,7 +512,6 @@ export function reconcileFinalBlockLogs( reconcileChildTraceSpans( updateConsole, workflowId, - log.blockId, childWorkflowInstanceId, executionId, log.childTraceSpans @@ -521,24 +520,34 @@ export function reconcileFinalBlockLogs( } } +/** + * Reconciles trace spans for blocks inside a child workflow. + * + * Inner-block console entries are created from SSE `block:started` events whose + * `childWorkflowBlockId` field carries the parent's per-invocation instanceId + * (see `execute/route.ts` where the server emits `childWorkflowContext.parentBlockId`). + * The matcher must therefore key on that instanceId — using the parent workflow + * block's static nodeId would never match and the rescue silently no-ops, leaving + * inner blocks stuck `isRunning: true` until `finishRunningEntries` sweeps them + * with a wall-clock duration. + */ function reconcileChildTraceSpans( updateConsole: UpdateConsoleFn, workflowId: string, - childWorkflowBlockId: string, childWorkflowInstanceId: string, executionId: string, spans: TraceSpan[] ): void { for (const span of spans) { const matchingEntry = span.blockId - ? findConsoleEntryForSpan(workflowId, executionId, childWorkflowBlockId, span) + ? findConsoleEntryForSpan(workflowId, executionId, childWorkflowInstanceId, span) : undefined if (span.blockId) { const errorMessage = normalizeSpanError(span.output?.error) updateConsole( span.blockId, { - ...spanConsoleIdentity(span, childWorkflowBlockId), + ...spanConsoleIdentity(span, childWorkflowInstanceId), replaceOutput: (span.output ?? {}) as Record, success: span.status !== 'error', ...(errorMessage !== undefined ? { error: errorMessage } : {}), @@ -555,7 +564,6 @@ function reconcileChildTraceSpans( reconcileChildTraceSpans( updateConsole, workflowId, - matchingEntry?.blockId ?? childWorkflowBlockId, matchingEntry?.childWorkflowInstanceId ?? childWorkflowInstanceId, executionId, span.children @@ -564,7 +572,7 @@ function reconcileChildTraceSpans( } } -function spanConsoleIdentity(span: TraceSpan, childWorkflowBlockId: string): ConsoleUpdate { +function spanConsoleIdentity(span: TraceSpan, childWorkflowInstanceId: string): ConsoleUpdate { const iterationContainerId = span.loopId ?? span.parallelId const iterationType = span.loopId ? 'loop' : span.parallelId ? 'parallel' : undefined return { @@ -573,18 +581,18 @@ function spanConsoleIdentity(span: TraceSpan, childWorkflowBlockId: string): Con ...(iterationType !== undefined && { iterationType }), ...(iterationContainerId !== undefined && { iterationContainerId }), ...(span.parentIterations !== undefined && { parentIterations: span.parentIterations }), - childWorkflowBlockId, + childWorkflowBlockId: childWorkflowInstanceId, } } function findConsoleEntryForSpan( workflowId: string, executionId: string, - childWorkflowBlockId: string, + childWorkflowInstanceId: string, span: TraceSpan ): ConsoleEntry | undefined { if (!span.blockId) return undefined - const identity = spanConsoleIdentity(span, childWorkflowBlockId) + const identity = spanConsoleIdentity(span, childWorkflowInstanceId) return useTerminalConsoleStore .getState() .getWorkflowEntries(workflowId) diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-editor/preview-editor.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-editor/preview-editor.tsx index 735408dae4b..e9d82206878 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-editor/preview-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/preview/components/preview-editor/preview-editor.tsx @@ -572,14 +572,12 @@ const SUBFLOW_CONFIG = { while: 'While Loop', doWhile: 'Do While Loop', }, - maxIterations: 1000, }, parallel: { typeLabels: { count: 'Parallel Count', collection: 'Parallel Each', }, - maxIterations: 20, }, } as const @@ -685,7 +683,7 @@ function SubflowConfigDisplay({ block, loop, parallel }: SubflowConfigDisplayPro className='mb-1' />
    - Enter a number between 1 and {config.maxIterations} + Enter a whole number greater than 0.
    ) : ( diff --git a/apps/sim/background/webhook-execution.ts b/apps/sim/background/webhook-execution.ts index bfd515695a3..1753813d849 100644 --- a/apps/sim/background/webhook-execution.ts +++ b/apps/sim/background/webhook-execution.ts @@ -578,7 +578,7 @@ async function executeWebhookJobInternal( snapshot, callbacks: {}, loggingSession, - includeFileBase64: true, + includeFileBase64: false, base64MaxBytes: undefined, abortSignal: timeoutController.signal, }) diff --git a/apps/sim/blocks/blocks/grafana.ts b/apps/sim/blocks/blocks/grafana.ts index 4ef7f36810a..1f65d255f79 100644 --- a/apps/sim/blocks/blocks/grafana.ts +++ b/apps/sim/blocks/blocks/grafana.ts @@ -126,6 +126,33 @@ Return ONLY the search query - no explanations, no quotes, no extra text.`, placeholder: 'tag1, tag2 (comma-separated)', condition: { field: 'operation', value: 'grafana_list_dashboards' }, }, + { + id: 'folderUIDs', + title: 'Folder UIDs', + type: 'short-input', + placeholder: 'uid1, uid2 (comma-separated)', + mode: 'advanced', + condition: { field: 'operation', value: 'grafana_list_dashboards' }, + }, + { + id: 'dashboardUIDs', + title: 'Dashboard UIDs', + type: 'short-input', + placeholder: 'uid1, uid2 (comma-separated)', + mode: 'advanced', + condition: { field: 'operation', value: 'grafana_list_dashboards' }, + }, + { + id: 'page', + title: 'Page', + type: 'short-input', + placeholder: 'Page number (1-based)', + mode: 'advanced', + condition: { + field: 'operation', + value: ['grafana_list_dashboards', 'grafana_list_folders'], + }, + }, // Create/Update Dashboard { @@ -156,13 +183,15 @@ Return ONLY the title - no explanations, no quotes, no extra text.`, id: 'folderUid', title: 'Folder UID', type: 'short-input', - placeholder: 'Optional - folder to create dashboard in', + placeholder: 'Folder UID (required for alert rules, optional for dashboards)', + required: { field: 'operation', value: 'grafana_create_alert_rule' }, condition: { field: 'operation', value: [ 'grafana_create_dashboard', 'grafana_update_dashboard', 'grafana_create_alert_rule', + 'grafana_update_alert_rule', ], }, }, @@ -229,6 +258,16 @@ Return ONLY the JSON array - no explanations, no markdown, no extra text.`, value: ['grafana_create_dashboard', 'grafana_update_dashboard'], }, }, + { + id: 'overwrite', + title: 'Overwrite on Conflict', + type: 'switch', + mode: 'advanced', + condition: { + field: 'operation', + value: ['grafana_create_dashboard', 'grafana_update_dashboard'], + }, + }, // Alert Rule operations { @@ -268,16 +307,6 @@ Return ONLY the alert title - no explanations, no quotes, no extra text.`, placeholder: 'Describe the alert...', }, }, - { - id: 'folderUid', - title: 'Folder UID', - type: 'short-input', - placeholder: 'Folder UID for the alert rule', - condition: { - field: 'operation', - value: ['grafana_create_alert_rule', 'grafana_update_alert_rule'], - }, - }, { id: 'ruleGroup', title: 'Rule Group', @@ -380,10 +409,105 @@ Return ONLY the JSON array - no explanations, no markdown, no extra text.`, title: 'Error State', type: 'dropdown', options: [ + { label: 'Error', id: 'Error' }, { label: 'Alerting', id: 'Alerting' }, { label: 'OK', id: 'OK' }, ], - value: () => 'Alerting', + value: () => 'Error', + condition: { + field: 'operation', + value: ['grafana_create_alert_rule', 'grafana_update_alert_rule'], + }, + }, + { + id: 'annotations', + title: 'Annotations (JSON)', + type: 'long-input', + placeholder: 'JSON object of alert annotations (e.g., {"summary":"..."})', + mode: 'advanced', + condition: { + field: 'operation', + value: ['grafana_create_alert_rule', 'grafana_update_alert_rule'], + }, + }, + { + id: 'labels', + title: 'Labels (JSON)', + type: 'long-input', + placeholder: 'JSON object of alert labels (e.g., {"severity":"critical"})', + mode: 'advanced', + condition: { + field: 'operation', + value: ['grafana_create_alert_rule', 'grafana_update_alert_rule'], + }, + }, + { + id: 'isPaused', + title: 'Paused', + type: 'switch', + mode: 'advanced', + condition: { + field: 'operation', + value: ['grafana_create_alert_rule', 'grafana_update_alert_rule'], + }, + }, + { + id: 'keepFiringFor', + title: 'Keep Firing For', + type: 'short-input', + placeholder: 'e.g., 5m', + mode: 'advanced', + condition: { + field: 'operation', + value: ['grafana_create_alert_rule', 'grafana_update_alert_rule'], + }, + }, + { + id: 'missingSeriesEvalsToResolve', + title: 'Missing Series Evals to Resolve', + type: 'short-input', + placeholder: 'e.g., 2', + mode: 'advanced', + condition: { + field: 'operation', + value: ['grafana_create_alert_rule', 'grafana_update_alert_rule'], + }, + }, + { + id: 'notificationSettings', + title: 'Notification Settings (JSON)', + type: 'long-input', + placeholder: 'JSON object of per-rule notification overrides', + mode: 'advanced', + condition: { + field: 'operation', + value: ['grafana_create_alert_rule', 'grafana_update_alert_rule'], + }, + }, + { + id: 'record', + title: 'Recording Rule (JSON)', + type: 'long-input', + placeholder: 'JSON object configuring this as a recording rule', + mode: 'advanced', + condition: { + field: 'operation', + value: ['grafana_create_alert_rule', 'grafana_update_alert_rule'], + }, + }, + { + id: 'alertRuleUidNew', + title: 'Custom Alert Rule UID', + type: 'short-input', + placeholder: 'Optional - auto-generated if not provided', + mode: 'advanced', + condition: { field: 'operation', value: 'grafana_create_alert_rule' }, + }, + { + id: 'disableProvenance', + title: 'Disable Provenance', + type: 'switch', + mode: 'advanced', condition: { field: 'operation', value: ['grafana_create_alert_rule', 'grafana_update_alert_rule'], @@ -396,7 +520,7 @@ Return ONLY the JSON array - no explanations, no markdown, no extra text.`, title: 'Annotation Text', type: 'long-input', placeholder: 'Enter annotation text...', - required: true, + required: { field: 'operation', value: 'grafana_create_annotation' }, condition: { field: 'operation', value: ['grafana_create_annotation', 'grafana_update_annotation'], @@ -436,8 +560,7 @@ Return ONLY the annotation text - no explanations, no quotes, no extra text.`, id: 'annotationDashboardUid', title: 'Dashboard UID', type: 'short-input', - placeholder: 'Enter dashboard UID', - required: true, + placeholder: 'Optional - omit for organization-wide annotations', condition: { field: 'operation', value: ['grafana_create_annotation', 'grafana_list_annotations'], @@ -453,6 +576,22 @@ Return ONLY the annotation text - no explanations, no quotes, no extra text.`, value: ['grafana_create_annotation', 'grafana_list_annotations'], }, }, + { + id: 'alertId', + title: 'Alert ID', + type: 'short-input', + placeholder: 'Filter by alert ID', + mode: 'advanced', + condition: { field: 'operation', value: 'grafana_list_annotations' }, + }, + { + id: 'userId', + title: 'User ID', + type: 'short-input', + placeholder: 'Filter by creator user ID', + mode: 'advanced', + condition: { field: 'operation', value: 'grafana_list_annotations' }, + }, { id: 'time', title: 'Time (epoch ms)', @@ -583,6 +722,30 @@ Return ONLY the folder title - no explanations, no quotes, no extra text.`, placeholder: 'Optional - auto-generated if not provided', condition: { field: 'operation', value: 'grafana_create_folder' }, }, + { + id: 'parentUidNew', + title: 'Parent Folder UID', + type: 'short-input', + placeholder: 'Optional - for nested folders', + mode: 'advanced', + condition: { field: 'operation', value: 'grafana_create_folder' }, + }, + { + id: 'parentUidList', + title: 'Parent Folder UID', + type: 'short-input', + placeholder: 'List children of this folder UID', + mode: 'advanced', + condition: { field: 'operation', value: 'grafana_list_folders' }, + }, + { + id: 'contactPointName', + title: 'Contact Point Name', + type: 'short-input', + placeholder: 'Filter by exact name', + mode: 'advanced', + condition: { field: 'operation', value: 'grafana_list_contact_points' }, + }, ], tools: { access: [ @@ -607,22 +770,30 @@ Return ONLY the folder title - no explanations, no quotes, no extra text.`, 'grafana_create_folder', ], config: { - tool: (params) => { - if (params.alertTitle) params.title = params.alertTitle - if (params.folderTitle) params.title = params.folderTitle - if (params.folderUidNew) params.uid = params.folderUidNew - if (params.annotationTags) params.tags = params.annotationTags - if (params.annotationDashboardUid) params.dashboardUid = params.annotationDashboardUid - return params.operation - }, + tool: (params) => params.operation, params: (params) => { const result: Record = {} + if (params.alertTitle) result.title = params.alertTitle + if (params.folderTitle) result.title = params.folderTitle + if (params.folderUidNew) result.uid = params.folderUidNew + if (params.alertRuleUidNew) result.uid = params.alertRuleUidNew + if (params.parentUidNew) result.parentUid = params.parentUidNew + if (params.parentUidList) result.parentUid = params.parentUidList + if (params.contactPointName) result.name = params.contactPointName + if (params.annotationTags) result.tags = params.annotationTags + if (params.annotationDashboardUid) result.dashboardUid = params.annotationDashboardUid if (params.panelId) result.panelId = Number(params.panelId) if (params.annotationId) result.annotationId = Number(params.annotationId) + if (params.alertId) result.alertId = Number(params.alertId) + if (params.userId) result.userId = Number(params.userId) if (params.time) result.time = Number(params.time) if (params.timeEnd) result.timeEnd = Number(params.timeEnd) if (params.from) result.from = Number(params.from) if (params.to) result.to = Number(params.to) + if (params.page) result.page = Number(params.page) + if (params.missingSeriesEvalsToResolve) { + result.missingSeriesEvalsToResolve = Number(params.missingSeriesEvalsToResolve) + } return result }, }, @@ -641,8 +812,15 @@ Return ONLY the folder title - no explanations, no quotes, no extra text.`, message: { type: 'string', description: 'Commit message' }, query: { type: 'string', description: 'Search query' }, tag: { type: 'string', description: 'Filter by tag' }, + folderUIDs: { + type: 'string', + description: 'Filter dashboards by folder UIDs (comma-separated)', + }, + dashboardUIDs: { type: 'string', description: 'Filter by dashboard UIDs (comma-separated)' }, + page: { type: 'number', description: 'Page number for pagination' }, // Alert inputs alertRuleUid: { type: 'string', description: 'Alert rule UID' }, + alertRuleUidNew: { type: 'string', description: 'Custom UID for newly created alert rule' }, alertTitle: { type: 'string', description: 'Alert rule title' }, ruleGroup: { type: 'string', description: 'Rule group name' }, condition: { type: 'string', description: 'Alert condition refId' }, @@ -650,14 +828,46 @@ Return ONLY the folder title - no explanations, no quotes, no extra text.`, forDuration: { type: 'string', description: 'Duration before firing' }, noDataState: { type: 'string', description: 'State on no data' }, execErrState: { type: 'string', description: 'State on error' }, + isPaused: { type: 'boolean', description: 'Whether the alert rule is paused' }, + keepFiringFor: { + type: 'string', + description: 'Duration to keep firing after the condition stops', + }, + missingSeriesEvalsToResolve: { + type: 'number', + description: 'Missing series evaluations before resolving', + }, + notificationSettings: { + type: 'string', + description: 'JSON of per-rule notification settings', + }, + record: { type: 'string', description: 'JSON of recording rule configuration' }, + disableProvenance: { + type: 'boolean', + description: 'Disable provenance tracking so the rule remains UI-editable', + }, + annotations: { type: 'string', description: 'JSON of alert annotations' }, + labels: { type: 'string', description: 'JSON of alert labels' }, + overwrite: { type: 'boolean', description: 'Overwrite existing dashboard on version conflict' }, // Annotation inputs text: { type: 'string', description: 'Annotation text' }, annotationId: { type: 'number', description: 'Annotation ID' }, + annotationTags: { type: 'string', description: 'Annotation tags (comma-separated)' }, + annotationDashboardUid: { type: 'string', description: 'Annotation dashboard UID' }, panelId: { type: 'number', description: 'Panel ID' }, time: { type: 'number', description: 'Start time (epoch ms)' }, timeEnd: { type: 'number', description: 'End time (epoch ms)' }, from: { type: 'number', description: 'Filter from time' }, to: { type: 'number', description: 'Filter to time' }, + alertId: { type: 'number', description: 'Filter annotations by alert ID' }, + userId: { type: 'number', description: 'Filter annotations by creator user ID' }, + // Folder inputs + folderTitle: { type: 'string', description: 'Folder title for newly created folder' }, + folderUidNew: { type: 'string', description: 'Custom UID for newly created folder' }, + parentUidList: { type: 'string', description: 'Parent folder UID to list children of' }, + parentUidNew: { type: 'string', description: 'Parent folder UID for newly created folder' }, + // Contact point inputs + contactPointName: { type: 'string', description: 'Filter contact points by name' }, // Data source inputs dataSourceId: { type: 'string', description: 'Data source ID or UID' }, }, @@ -675,6 +885,26 @@ Return ONLY the folder title - no explanations, no quotes, no extra text.`, // Alert outputs rules: { type: 'json', description: 'Alert rules list' }, contactPoints: { type: 'json', description: 'Contact points list' }, + condition: { type: 'string', description: 'Alert condition refId' }, + for: { type: 'string', description: 'Duration the condition must hold before firing' }, + keepFiringFor: { + type: 'string', + description: 'Duration to keep firing after the condition stops', + }, + missingSeriesEvalsToResolve: { + type: 'number', + description: 'Missing series evaluations before resolving', + }, + isPaused: { type: 'boolean', description: 'Whether the alert rule is paused' }, + folderUID: { type: 'string', description: 'Parent folder UID' }, + ruleGroup: { type: 'string', description: 'Rule group name' }, + orgID: { type: 'number', description: 'Organization ID' }, + provenance: { type: 'string', description: 'Provisioning source' }, + noDataState: { type: 'string', description: 'State on no data' }, + execErrState: { type: 'string', description: 'State on execution error' }, + notification_settings: { type: 'json', description: 'Per-rule notification settings' }, + record: { type: 'json', description: 'Recording rule configuration' }, + updated: { type: 'string', description: 'Last update timestamp' }, // Annotation outputs annotations: { type: 'json', description: 'Annotations list' }, id: { type: 'number', description: 'Annotation ID' }, diff --git a/apps/sim/blocks/blocks/mothership.ts b/apps/sim/blocks/blocks/mothership.ts index 9b1818667ea..a524eb1674f 100644 --- a/apps/sim/blocks/blocks/mothership.ts +++ b/apps/sim/blocks/blocks/mothership.ts @@ -6,6 +6,7 @@ interface MothershipResponse extends ToolResponse { output: { content: string model: string + conversationId?: string tokens?: { prompt?: number completion?: number @@ -34,6 +35,12 @@ export const MothershipBlock: BlockConfig = { type: 'long-input', placeholder: 'Enter your prompt for the Mothership...', }, + { + id: 'conversationId', + title: 'Conversation ID', + type: 'short-input', + placeholder: 'e.g., user-123, session-abc, customer-456', + }, ], tools: { access: [], @@ -43,10 +50,15 @@ export const MothershipBlock: BlockConfig = { type: 'string', description: 'The prompt to send to the Mothership AI agent', }, + conversationId: { + type: 'string', + description: 'Mothership chat ID to continue; generated when omitted', + }, }, outputs: { content: { type: 'string', description: 'Generated response content' }, model: { type: 'string', description: 'Model used for generation' }, + conversationId: { type: 'string', description: 'Mothership chat ID used for this request' }, tokens: { type: 'json', description: 'Token usage statistics' }, toolCalls: { type: 'json', description: 'Tool calls made during execution' }, cost: { type: 'json', description: 'Cost of the execution' }, diff --git a/apps/sim/blocks/blocks/workflow.ts b/apps/sim/blocks/blocks/workflow.ts index 4cc67230a3a..667b6614ea4 100644 --- a/apps/sim/blocks/blocks/workflow.ts +++ b/apps/sim/blocks/blocks/workflow.ts @@ -14,9 +14,20 @@ export const WorkflowBlock: BlockConfig = { id: 'workflowId', title: 'Select Workflow', type: 'workflow-selector', + canonicalParamId: 'workflowId', selectorKey: 'sim.workflows', placeholder: 'Search workflows...', required: true, + mode: 'basic', + }, + { + id: 'manualWorkflowId', + title: 'Workflow ID', + type: 'short-input', + canonicalParamId: 'workflowId', + placeholder: 'Enter workflow ID', + required: true, + mode: 'advanced', }, { id: 'input', diff --git a/apps/sim/blocks/blocks/workflow_input.ts b/apps/sim/blocks/blocks/workflow_input.ts index 62db4197f3d..213124300e6 100644 --- a/apps/sim/blocks/blocks/workflow_input.ts +++ b/apps/sim/blocks/blocks/workflow_input.ts @@ -19,9 +19,20 @@ export const WorkflowInputBlock: BlockConfig = { id: 'workflowId', title: 'Select Workflow', type: 'workflow-selector', + canonicalParamId: 'workflowId', selectorKey: 'sim.workflows', placeholder: 'Search workflows...', required: true, + mode: 'basic', + }, + { + id: 'manualWorkflowId', + title: 'Workflow ID', + type: 'short-input', + canonicalParamId: 'workflowId', + placeholder: 'Enter workflow ID', + required: true, + mode: 'advanced', }, { id: 'inputMapping', diff --git a/apps/sim/executor/constants.ts b/apps/sim/executor/constants.ts index feb962fe9ce..71fa8dea3f1 100644 --- a/apps/sim/executor/constants.ts +++ b/apps/sim/executor/constants.ts @@ -67,6 +67,7 @@ export const EDGE = { LOOP_CONTINUE: 'loop_continue', LOOP_CONTINUE_ALT: 'loop-continue-source', LOOP_EXIT: 'loop_exit', + PARALLEL_CONTINUE: 'parallel_continue', PARALLEL_EXIT: 'parallel_exit', ERROR: 'error', SOURCE: 'source', @@ -158,8 +159,7 @@ export const DEFAULTS = { BLOCK_TYPE: 'unknown', BLOCK_TITLE: 'Untitled Block', WORKFLOW_NAME: 'Workflow', - MAX_LOOP_ITERATIONS: 1000, - MAX_FOREACH_ITEMS: 1000, + DEFAULT_LOOP_ITERATIONS: 1000, MAX_PARALLEL_BRANCHES: 20, MAX_NESTING_DEPTH: 10, /** Maximum child workflow depth for propagating SSE callbacks (block:started, block:completed). */ diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index f1506bb33fc..a8fd36c63a7 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -3,6 +3,7 @@ import { toError } from '@sim/utils/errors' import { redactApiKeys } from '@/lib/core/security/redaction' import { normalizeStringArray } from '@/lib/core/utils/arrays' import { getBaseUrl } from '@/lib/core/utils/urls' +import { compactExecutionPayload } from '@/lib/execution/payloads/serializer' import { containsUserFileWithMetadata, hydrateUserFilesWithBase64, @@ -126,7 +127,12 @@ export class BlockExecutor { resolvedInputs: fnInputs, displayInputs, contextVariables, - } = this.resolver.resolveInputsForFunctionBlock(ctx, node.id, block.config.params, block) + } = await this.resolver.resolveInputsForFunctionBlock( + ctx, + node.id, + block.config.params, + block + ) resolvedInputs = { ...fnInputs, [FUNCTION_BLOCK_CONTEXT_VARS_KEY]: contextVariables, @@ -136,7 +142,7 @@ export class BlockExecutor { } inputsForLog = displayInputs } else { - resolvedInputs = this.resolver.resolveInputs(ctx, node.id, block.config.params, block) + resolvedInputs = await this.resolver.resolveInputs(ctx, node.id, block.config.params, block) inputsForLog = resolvedInputs } @@ -189,14 +195,28 @@ export class BlockExecutor { normalizedOutput = this.normalizeOutput(output) } - if (containsUserFileWithMetadata(normalizedOutput)) { + if (ctx.includeFileBase64 === true && containsUserFileWithMetadata(normalizedOutput)) { normalizedOutput = (await hydrateUserFilesWithBase64(normalizedOutput, { requestId: ctx.metadata.requestId, + workspaceId: ctx.workspaceId, + workflowId: ctx.workflowId, executionId: ctx.executionId, + largeValueExecutionIds: ctx.largeValueExecutionIds, + allowLargeValueWorkflowScope: ctx.allowLargeValueWorkflowScope, + userId: ctx.userId, maxBytes: ctx.base64MaxBytes, })) as NormalizedBlockOutput } + normalizedOutput = (await compactExecutionPayload(normalizedOutput, { + workspaceId: ctx.workspaceId, + workflowId: ctx.workflowId, + executionId: ctx.executionId, + userId: ctx.userId, + preserveUserFileBase64: ctx.includeFileBase64 === true, + requireDurable: true, + })) as NormalizedBlockOutput + const endedAt = new Date().toISOString() const duration = performance.now() - startTime diff --git a/apps/sim/executor/execution/edge-manager.ts b/apps/sim/executor/execution/edge-manager.ts index 7bedea3a5a2..63a0748c8c6 100644 --- a/apps/sim/executor/execution/edge-manager.ts +++ b/apps/sim/executor/execution/edge-manager.ts @@ -230,6 +230,10 @@ export class EdgeManager { return handle === EDGE.PARALLEL_EXIT } + if (output.selectedRoute === EDGE.PARALLEL_CONTINUE) { + return false + } + if (!handle) { return true } diff --git a/apps/sim/executor/execution/executor.ts b/apps/sim/executor/execution/executor.ts index a141e017fb1..4866ebeba81 100644 --- a/apps/sim/executor/execution/executor.ts +++ b/apps/sim/executor/execution/executor.ts @@ -18,6 +18,7 @@ import { LoopOrchestrator } from '@/executor/orchestrators/loop' import { NodeExecutionOrchestrator } from '@/executor/orchestrators/node' import { ParallelOrchestrator } from '@/executor/orchestrators/parallel' import type { BlockState, ExecutionContext, ExecutionResult } from '@/executor/types' +import { ParallelExpander } from '@/executor/utils/parallel-expansion' import { computeExecutionSets, type RunFromBlockContext, @@ -34,6 +35,7 @@ import { extractParallelIdFromSentinel, } from '@/executor/utils/subflow-utils' import { VariableResolver } from '@/executor/variables/resolver' +import { navigatePathAsync } from '@/executor/variables/resolvers/reference-async.server' import type { SerializedWorkflow } from '@/serializer/types' import type { SubflowType } from '@/stores/workflows/workflow/types' @@ -78,6 +80,8 @@ export class DAGExecutor { triggerBlockId, savedIncomingEdges, }) + this.restoreSnapshotParallelBatches(dag, this.contextExtensions.snapshotState) + this.restoreSavedIncomingEdges(dag, savedIncomingEdges) const { context, state } = this.createExecutionContext(workflowId, triggerBlockId) context.subflowParentMap = this.buildSubflowParentMap(dag) @@ -212,8 +216,45 @@ export class DAGExecutor { return await engine.run() } + private restoreSavedIncomingEdges(dag: DAG, savedIncomingEdges?: Record): void { + if (!savedIncomingEdges) return + + for (const [nodeId, incomingEdges] of Object.entries(savedIncomingEdges)) { + const node = dag.nodes.get(nodeId) + if (node) { + node.incomingEdges = new Set(incomingEdges) + } + } + } + + private restoreSnapshotParallelBatches( + dag: DAG, + snapshotState?: SerializableExecutionState + ): void { + if (!snapshotState?.parallelExecutions) return + + const expander = new ParallelExpander() + for (const [parallelId, scope] of Object.entries(snapshotState.parallelExecutions)) { + const currentBatchSize = Number(scope.currentBatchSize ?? 0) + if (!Number.isFinite(currentBatchSize) || currentBatchSize <= 0) continue + + const currentBatchStart = Number(scope.currentBatchStart ?? 0) + const totalBranches = Number(scope.totalBranches ?? currentBatchStart + currentBatchSize) + const items = Array.isArray(scope.items) + ? scope.items.slice(currentBatchStart, currentBatchStart + currentBatchSize) + : undefined + + expander.expandParallel(dag, parallelId, currentBatchSize, items, { + branchIndexOffset: currentBatchStart, + totalBranches, + }) + } + } + private buildExecutionPipeline(context: ExecutionContext, dag: DAG, state: ExecutionState) { - const resolver = new VariableResolver(this.workflow, this.workflowVariables, state) + const resolver = new VariableResolver(this.workflow, this.workflowVariables, state, { + navigatePathAsync, + }) const allHandlers = createBlockHandlers() const blockExecutor = new BlockExecutor(allHandlers, resolver, this.contextExtensions, state) const edgeManager = new EdgeManager(dag) @@ -271,6 +312,8 @@ export class DAGExecutor { workflowId, workspaceId: this.contextExtensions.workspaceId, executionId: this.contextExtensions.executionId, + largeValueExecutionIds: this.contextExtensions.largeValueExecutionIds, + allowLargeValueWorkflowScope: this.contextExtensions.allowLargeValueWorkflowScope, userId: this.contextExtensions.userId, isDeployedContext: this.contextExtensions.isDeployedContext, enforceCredentialAccess: this.contextExtensions.enforceCredentialAccess, @@ -317,10 +360,18 @@ export class DAGExecutor { branchOutputs: scope.branchOutputs ? new Map(Object.entries(scope.branchOutputs).map(([k, v]) => [Number(k), v])) : new Map(), + accumulatedOutputs: scope.accumulatedOutputs + ? new Map( + Object.entries(scope.accumulatedOutputs).map(([k, v]) => [Number(k), v]) + ) + : new Map(), }, ]) ) : new Map(), + parallelBlockMapping: snapshotState?.parallelBlockMapping + ? new Map(Object.entries(snapshotState.parallelBlockMapping)) + : new Map(), executedBlocks: state.getExecutedBlocks(), activeExecutionPath: snapshotState?.activeExecutionPath ? new Set(snapshotState.activeExecutionPath) diff --git a/apps/sim/executor/execution/snapshot-serializer.test.ts b/apps/sim/executor/execution/snapshot-serializer.test.ts new file mode 100644 index 00000000000..9aa273d2bbd --- /dev/null +++ b/apps/sim/executor/execution/snapshot-serializer.test.ts @@ -0,0 +1,70 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { serializePauseSnapshot } from '@/executor/execution/snapshot-serializer' +import type { ExecutionContext } from '@/executor/types' + +function createContext(overrides: Partial = {}): ExecutionContext { + return { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', + userId: 'user-1', + blockStates: new Map(), + executedBlocks: new Set(), + blockLogs: [], + metadata: { + requestId: 'request-1', + executionId: 'execution-1', + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + userId: 'user-1', + triggerType: 'manual', + useDraftState: true, + startTime: '2026-01-01T00:00:00.000Z', + }, + environmentVariables: {}, + decisions: { + router: new Map(), + condition: new Map(), + }, + completedLoops: new Set(), + activeExecutionPath: new Set(), + ...overrides, + } as ExecutionContext +} + +describe('serializePauseSnapshot', () => { + it('serializes batched parallel accumulated outputs for cross-process resume', () => { + const context = createContext({ + parallelExecutions: new Map([ + [ + 'parallel-1', + { + parallelId: 'parallel-1', + totalBranches: 3, + branchOutputs: new Map([[2, [{ output: 'current-batch' }]]]), + accumulatedOutputs: new Map([ + [0, [{ output: 'batch-0' }]], + [1, [{ output: 'batch-1' }]], + ]), + }, + ], + ]), + }) + + const snapshot = serializePauseSnapshot(context, ['next-block']) + const serialized = JSON.parse(snapshot.snapshot) + + expect(serialized.state.parallelExecutions?.['parallel-1']).toMatchObject({ + branchOutputs: { + 2: [{ output: 'current-batch' }], + }, + accumulatedOutputs: { + 0: [{ output: 'batch-0' }], + 1: [{ output: 'batch-1' }], + }, + }) + }) +}) diff --git a/apps/sim/executor/execution/snapshot-serializer.ts b/apps/sim/executor/execution/snapshot-serializer.ts index 76c2a3dba5f..fbac5b893c2 100644 --- a/apps/sim/executor/execution/snapshot-serializer.ts +++ b/apps/sim/executor/execution/snapshot-serializer.ts @@ -35,16 +35,19 @@ function serializeParallelExecutions( if (!parallelExecutions) return undefined const result: Record = {} for (const [parallelId, scope] of parallelExecutions.entries()) { - let branchOutputs: any - if (scope.branchOutputs instanceof Map) { - branchOutputs = Object.fromEntries(scope.branchOutputs) - } else { - branchOutputs = scope.branchOutputs ?? {} - } + const branchOutputs = + scope.branchOutputs instanceof Map + ? Object.fromEntries(scope.branchOutputs) + : (scope.branchOutputs ?? {}) + const accumulatedOutputs = + scope.accumulatedOutputs instanceof Map + ? Object.fromEntries(scope.accumulatedOutputs) + : (scope.accumulatedOutputs ?? {}) result[parallelId] = { ...scope, branchOutputs, + accumulatedOutputs, } } return result diff --git a/apps/sim/executor/execution/state.ts b/apps/sim/executor/execution/state.ts index f9a664ca309..eefe09338c2 100644 --- a/apps/sim/executor/execution/state.ts +++ b/apps/sim/executor/execution/state.ts @@ -21,6 +21,10 @@ export interface LoopScope { export interface ParallelScope { parallelId: string totalBranches: number + batchSize?: number + currentBatchStart?: number + currentBatchSize?: number + accumulatedOutputs?: Map branchOutputs: Map items?: any[] /** Error message if parallel validation failed (e.g., exceeded max branches) */ diff --git a/apps/sim/executor/execution/types.ts b/apps/sim/executor/execution/types.ts index 1ed3db20a32..0180d9e0ad7 100644 --- a/apps/sim/executor/execution/types.ts +++ b/apps/sim/executor/execution/types.ts @@ -35,6 +35,8 @@ export interface ExecutionMetadata { parallels?: Record deploymentVersionId?: string } + largeValueExecutionIds?: string[] + allowLargeValueWorkflowScope?: boolean callChain?: string[] correlation?: AsyncExecutionCorrelation executionMode?: 'sync' | 'stream' | 'async' @@ -143,6 +145,8 @@ export interface ExecutionCallbacks { export interface ContextExtensions { workspaceId?: string executionId?: string + largeValueExecutionIds?: string[] + allowLargeValueWorkflowScope?: boolean userId?: string stream?: boolean selectedOutputs?: string[] diff --git a/apps/sim/executor/handlers/function/function-handler.test.ts b/apps/sim/executor/handlers/function/function-handler.test.ts index b288940850d..aafd49faea5 100644 --- a/apps/sim/executor/handlers/function/function-handler.test.ts +++ b/apps/sim/executor/handlers/function/function-handler.test.ts @@ -81,6 +81,7 @@ describe('FunctionBlockHandler', () => { _context: { workflowId: mockContext.workflowId, workspaceId: mockContext.workspaceId, + executionId: mockContext.executionId, userId: mockContext.userId, isDeployedContext: mockContext.isDeployedContext, enforceCredentialAccess: mockContext.enforceCredentialAccess, @@ -121,6 +122,7 @@ describe('FunctionBlockHandler', () => { _context: { workflowId: mockContext.workflowId, workspaceId: mockContext.workspaceId, + executionId: mockContext.executionId, userId: mockContext.userId, isDeployedContext: mockContext.isDeployedContext, enforceCredentialAccess: mockContext.enforceCredentialAccess, @@ -154,6 +156,7 @@ describe('FunctionBlockHandler', () => { _context: { workflowId: mockContext.workflowId, workspaceId: mockContext.workspaceId, + executionId: mockContext.executionId, userId: mockContext.userId, isDeployedContext: mockContext.isDeployedContext, enforceCredentialAccess: mockContext.enforceCredentialAccess, diff --git a/apps/sim/executor/handlers/function/function-handler.ts b/apps/sim/executor/handlers/function/function-handler.ts index 53fa8b4451b..ec08996ba5b 100644 --- a/apps/sim/executor/handlers/function/function-handler.ts +++ b/apps/sim/executor/handlers/function/function-handler.ts @@ -49,7 +49,7 @@ export class FunctionBlockHandler implements BlockHandler { readCodeContent(inputs[FUNCTION_BLOCK_DISPLAY_CODE_KEY]) ?? readCodeContent((block.config?.params as Record | undefined)?.code) - const { blockData, blockNameMapping, blockOutputSchemas } = collectBlockData(ctx) + const { blockNameMapping, blockOutputSchemas } = collectBlockData(ctx) const contextVariables = normalizeRecord(inputs[FUNCTION_BLOCK_CONTEXT_VARS_KEY]) @@ -60,13 +60,16 @@ export class FunctionBlockHandler implements BlockHandler { timeout: inputs.timeout || DEFAULT_EXECUTION_TIMEOUT_MS, envVars: normalizeStringRecord(ctx.environmentVariables), workflowVariables: normalizeWorkflowVariables(ctx.workflowVariables), - blockData, + blockData: {}, blockNameMapping, blockOutputSchemas, contextVariables, _context: { workflowId: ctx.workflowId, workspaceId: ctx.workspaceId, + executionId: ctx.executionId, + largeValueExecutionIds: ctx.largeValueExecutionIds, + allowLargeValueWorkflowScope: ctx.allowLargeValueWorkflowScope, userId: ctx.userId, isDeployedContext: ctx.isDeployedContext, enforceCredentialAccess: ctx.enforceCredentialAccess, diff --git a/apps/sim/executor/handlers/mothership/mothership-handler.test.ts b/apps/sim/executor/handlers/mothership/mothership-handler.test.ts index 38a36c32f0e..1dde2fcebc0 100644 --- a/apps/sim/executor/handlers/mothership/mothership-handler.test.ts +++ b/apps/sim/executor/handlers/mothership/mothership-handler.test.ts @@ -84,7 +84,7 @@ describe('MothershipBlockHandler', () => { metadata: { id: BlockType.MOTHERSHIP, name: 'Mothership' }, position: { x: 0, y: 0 }, config: { tool: BlockType.MOTHERSHIP, params: {} }, - inputs: { prompt: 'string' }, + inputs: { prompt: 'string', conversationId: 'string' }, outputs: {}, enabled: true, } as SerializedBlock @@ -122,6 +122,7 @@ describe('MothershipBlockHandler', () => { JSON.stringify({ content: 'done', model: 'mothership', + conversationId: 'chat-uuid', tokens: { total: 5 }, toolCalls: [], }), @@ -137,6 +138,7 @@ describe('MothershipBlockHandler', () => { expect(result).toEqual({ content: 'done', model: 'mothership', + conversationId: 'chat-uuid', tokens: { total: 5 }, toolCalls: { list: [], count: 0 }, cost: undefined, @@ -161,6 +163,55 @@ describe('MothershipBlockHandler', () => { }) }) + it('uses a provided conversation ID as the mothership chat ID', async () => { + mockGenerateId.mockReturnValueOnce('message-uuid') + mockGenerateId.mockReturnValueOnce('request-uuid') + + fetchMock.mockResolvedValue( + new Response( + JSON.stringify({ + content: 'continued', + model: 'mothership', + conversationId: 'existing-chat-id', + tokens: {}, + toolCalls: [], + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' }, + } + ) + ) + + const result = await handler.execute(context, block, { + prompt: 'Continue this thread', + conversationId: ' existing-chat-id ', + }) + + expect(result).toEqual({ + content: 'continued', + model: 'mothership', + conversationId: 'existing-chat-id', + tokens: {}, + toolCalls: { list: [], count: 0 }, + cost: undefined, + }) + + const [, options] = fetchMock.mock.calls[0] as [string, RequestInit] + const body = JSON.parse(String(options.body)) + expect(body).toEqual({ + messages: [{ role: 'user', content: 'Continue this thread' }], + workspaceId: 'workspace-1', + userId: 'user-1', + chatId: 'existing-chat-id', + messageId: 'message-uuid', + requestId: 'request-uuid', + workflowId: 'workflow-1', + executionId: 'execution-1', + }) + expect(mockGenerateId).toHaveBeenCalledTimes(2) + }) + it('propagates local aborts to the mothership request', async () => { const abortController = new AbortController() context.abortSignal = abortController.signal diff --git a/apps/sim/executor/handlers/mothership/mothership-handler.ts b/apps/sim/executor/handlers/mothership/mothership-handler.ts index 433bcc273e2..3c48d746909 100644 --- a/apps/sim/executor/handlers/mothership/mothership-handler.ts +++ b/apps/sim/executor/handlers/mothership/mothership-handler.ts @@ -33,7 +33,9 @@ export class MothershipBlockHandler implements BlockHandler { throw new Error('Prompt input is required') } const messages = [{ role: 'user' as const, content: prompt }] - const chatId = generateId() + const providedConversationId = + typeof inputs.conversationId === 'string' ? inputs.conversationId.trim() : '' + const chatId = providedConversationId || generateId() const messageId = generateId() const requestId = generateId() @@ -57,6 +59,7 @@ export class MothershipBlockHandler implements BlockHandler { requestId, workflowId: ctx.workflowId, executionId: ctx.executionId, + chatId, }) const abortController = new AbortController() @@ -135,6 +138,7 @@ export class MothershipBlockHandler implements BlockHandler { return { content: result.content || '', model: result.model || 'mothership', + conversationId: result.conversationId || chatId, tokens: result.tokens || {}, toolCalls, cost: result.cost || undefined, diff --git a/apps/sim/executor/orchestrators/loop.ts b/apps/sim/executor/orchestrators/loop.ts index 1c089ac3cb3..2087bf09c45 100644 --- a/apps/sim/executor/orchestrators/loop.ts +++ b/apps/sim/executor/orchestrators/loop.ts @@ -3,6 +3,8 @@ import { toError } from '@sim/utils/errors' import { generateRequestId } from '@/lib/core/utils/request' import { isExecutionCancelled, isRedisCancellationEnabled } from '@/lib/execution/cancellation' import { executeInIsolatedVM } from '@/lib/execution/isolated-vm' +import { compactSubflowResults } from '@/lib/execution/payloads/serializer' +import { isLikelyReferenceSegment } from '@/lib/workflows/sanitization/references' import { buildLoopIndexCondition, DEFAULTS, EDGE, PARALLEL } from '@/executor/constants' import type { DAG } from '@/executor/dag/builder' import type { EdgeManager } from '@/executor/execution/edge-manager' @@ -10,7 +12,7 @@ import type { LoopScope } from '@/executor/execution/state' import type { BlockStateController, ContextExtensions } from '@/executor/execution/types' import type { ExecutionContext, NormalizedBlockOutput } from '@/executor/types' import type { LoopConfigWithNodes } from '@/executor/types/loop' -import { replaceValidReferences } from '@/executor/utils/reference-validation' +import { createReferencePattern } from '@/executor/utils/reference-validation' import { addSubflowErrorLog, buildParallelSentinelEndId, @@ -20,8 +22,7 @@ import { emitEmptySubflowEvents, emitSubflowSuccessEvents, extractBaseBlockId, - resolveArrayInput, - validateMaxCount, + resolveArrayInputAsync, } from '@/executor/utils/subflow-utils' import type { VariableResolver } from '@/executor/variables/resolver' import type { SerializedLoop } from '@/serializer/types' @@ -30,13 +31,31 @@ const logger = createLogger('LoopOrchestrator') const LOOP_CONDITION_TIMEOUT_MS = 5000 +async function replaceLoopConditionReferences( + condition: string, + replacer: (match: string) => Promise +): Promise { + const pattern = createReferencePattern() + let cursor = 0 + let result = '' + for (const match of condition.matchAll(pattern)) { + const fullMatch = match[0] + const index = match.index ?? 0 + result += condition.slice(cursor, index) + result += isLikelyReferenceSegment(fullMatch) ? await replacer(fullMatch) : fullMatch + cursor = index + fullMatch.length + } + return result + condition.slice(cursor) +} + export type LoopRoute = typeof EDGE.LOOP_CONTINUE | typeof EDGE.LOOP_EXIT export interface LoopContinuationResult { shouldContinue: boolean shouldExit: boolean selectedRoute: LoopRoute - aggregatedResults?: NormalizedBlockOutput[][] + aggregatedResults?: unknown + totalIterations?: number } export class LoopOrchestrator { @@ -87,25 +106,7 @@ export class LoopOrchestrator { switch (loopType) { case 'for': { scope.loopType = 'for' - const requestedIterations = loopConfig.iterations || DEFAULTS.MAX_LOOP_ITERATIONS - - const iterationError = validateMaxCount( - requestedIterations, - DEFAULTS.MAX_LOOP_ITERATIONS, - 'For loop iterations' - ) - if (iterationError) { - logger.error(iterationError, { loopId, requestedIterations }) - await this.addLoopErrorLog(ctx, loopId, loopType, iterationError, { - iterations: requestedIterations, - }) - scope.maxIterations = 0 - scope.validationError = iterationError - scope.condition = buildLoopIndexCondition(0) - ctx.loopExecutions?.set(loopId, scope) - throw new Error(iterationError) - } - + const requestedIterations = loopConfig.iterations || DEFAULTS.DEFAULT_LOOP_ITERATIONS scope.maxIterations = requestedIterations scope.condition = buildLoopIndexCondition(scope.maxIterations) break @@ -133,7 +134,7 @@ export class LoopOrchestrator { } let items: any[] try { - items = resolveArrayInput(ctx, loopConfig.forEachItems, this.resolver) + items = await resolveArrayInputAsync(ctx, loopConfig.forEachItems, this.resolver) } catch (error) { const errorMessage = `ForEach loop resolution failed: ${toError(error).message}` logger.error(errorMessage, { loopId, forEachItems: loopConfig.forEachItems }) @@ -148,25 +149,6 @@ export class LoopOrchestrator { throw new Error(errorMessage) } - const sizeError = validateMaxCount( - items.length, - DEFAULTS.MAX_FOREACH_ITEMS, - 'ForEach loop collection size' - ) - if (sizeError) { - logger.error(sizeError, { loopId, collectionSize: items.length }) - await this.addLoopErrorLog(ctx, loopId, loopType, sizeError, { - forEachItems: loopConfig.forEachItems, - collectionSize: items.length, - }) - scope.items = [] - scope.maxIterations = 0 - scope.validationError = sizeError - scope.condition = buildLoopIndexCondition(0) - ctx.loopExecutions?.set(loopId, scope) - throw new Error(sizeError) - } - scope.items = items scope.maxIterations = items.length scope.item = items[0] @@ -184,25 +166,7 @@ export class LoopOrchestrator { if (loopConfig.doWhileCondition) { scope.condition = loopConfig.doWhileCondition } else { - const requestedIterations = loopConfig.iterations || DEFAULTS.MAX_LOOP_ITERATIONS - - const iterationError = validateMaxCount( - requestedIterations, - DEFAULTS.MAX_LOOP_ITERATIONS, - 'Do-While loop iterations' - ) - if (iterationError) { - logger.error(iterationError, { loopId, requestedIterations }) - await this.addLoopErrorLog(ctx, loopId, loopType, iterationError, { - iterations: requestedIterations, - }) - scope.maxIterations = 0 - scope.validationError = iterationError - scope.condition = buildLoopIndexCondition(0) - ctx.loopExecutions?.set(loopId, scope) - throw new Error(iterationError) - } - + const requestedIterations = loopConfig.iterations || DEFAULTS.DEFAULT_LOOP_ITERATIONS scope.maxIterations = requestedIterations scope.condition = buildLoopIndexCondition(scope.maxIterations) } @@ -313,8 +277,17 @@ export class LoopOrchestrator { scope: LoopScope ): Promise { const results = scope.allIterationOutputs - const output = { results } + const totalIterations = results.length + const compactedResults = await compactSubflowResults(results, { + workspaceId: ctx.workspaceId, + workflowId: ctx.workflowId, + executionId: ctx.executionId, + userId: ctx.userId, + requireDurable: true, + }) + const output = { results: compactedResults } this.state.setBlockOutput(loopId, output, DEFAULTS.EXECUTION_TIME) + scope.allIterationOutputs = [] await emitSubflowSuccessEvents(ctx, loopId, 'loop', output, this.contextExtensions) @@ -322,7 +295,8 @@ export class LoopOrchestrator { shouldContinue: false, shouldExit: true, selectedRoute: EDGE.LOOP_EXIT, - aggregatedResults: results, + aggregatedResults: output.results, + totalIterations, } } @@ -680,8 +654,8 @@ export class LoopOrchestrator { workflowVariables: ctx.workflowVariables, }) - const evaluatedCondition = replaceValidReferences(condition, (match) => { - const resolved = this.resolver.resolveSingleReference(ctx, '', match, scope) + const evaluatedCondition = await replaceLoopConditionReferences(condition, async (match) => { + const resolved = await this.resolver.resolveSingleReference(ctx, '', match, scope) logger.debug('Resolved variable reference in loop condition', { reference: match, resolvedValue: resolved, diff --git a/apps/sim/executor/orchestrators/node.ts b/apps/sim/executor/orchestrators/node.ts index 9844e93fb57..4db656c3254 100644 --- a/apps/sim/executor/orchestrators/node.ts +++ b/apps/sim/executor/orchestrators/node.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref' import { EDGE } from '@/executor/constants' import type { DAG, DAGNode } from '@/executor/dag/builder' import type { BlockExecutor } from '@/executor/execution/block-executor' @@ -10,6 +11,20 @@ import { extractBaseBlockId } from '@/executor/utils/subflow-utils' const logger = createLogger('NodeExecutionOrchestrator') +function getResultCount(value: unknown): number { + if (isLargeValueRef(value)) { + const preview = value.preview + if ( + preview && + typeof preview === 'object' && + typeof (preview as Record).length === 'number' + ) { + return (preview as { length: number }).length + } + } + return Array.isArray(value) ? value.length : 0 +} + export interface NodeExecutionResult { nodeId: string output: NormalizedBlockOutput @@ -130,7 +145,9 @@ export class NodeExecutionOrchestrator { shouldContinue: false, shouldExit: true, selectedRoute: continuationResult.selectedRoute, - totalIterations: continuationResult.aggregatedResults?.length || 0, + totalIterations: + continuationResult.totalIterations ?? + getResultCount(continuationResult.aggregatedResults), } } @@ -174,6 +191,14 @@ export class NodeExecutionOrchestrator { if (sentinelType === 'end') { const result = await this.parallelOrchestrator.aggregateParallelResults(ctx, parallelId) + if (!result.allBranchesComplete) { + return { + results: [], + sentinelEnd: true, + selectedRoute: EDGE.PARALLEL_CONTINUE, + totalBranches: result.totalBranches, + } + } return { results: result.results || [], sentinelEnd: true, @@ -258,6 +283,14 @@ export class NodeExecutionOrchestrator { this.loopOrchestrator.restoreLoopEdges(loopId) } } + + if ( + node.metadata.isParallelSentinel && + node.metadata.sentinelType === 'end' && + output.selectedRoute === EDGE.PARALLEL_CONTINUE + ) { + this.state.deleteBlockState(node.id) + } } private findParallelIdForNode(nodeId: string): string | undefined { diff --git a/apps/sim/executor/orchestrators/parallel.test.ts b/apps/sim/executor/orchestrators/parallel.test.ts index f0262b92e9e..96aa1ae684d 100644 --- a/apps/sim/executor/orchestrators/parallel.test.ts +++ b/apps/sim/executor/orchestrators/parallel.test.ts @@ -6,6 +6,15 @@ import type { DAG } from '@/executor/dag/builder' import type { BlockStateWriter, ContextExtensions } from '@/executor/execution/types' import { ParallelOrchestrator } from '@/executor/orchestrators/parallel' import type { ExecutionContext } from '@/executor/types' +import { buildBranchNodeId } from '@/executor/utils/subflow-utils' + +const { mockCompactSubflowResults } = vi.hoisted(() => ({ + mockCompactSubflowResults: vi.fn(async (results: unknown) => results), +})) + +vi.mock('@/lib/execution/payloads/serializer', () => ({ + compactSubflowResults: mockCompactSubflowResults, +})) function createDag(): DAG { return { @@ -75,6 +84,7 @@ function createContext(overrides: Partial = {}): ExecutionCont describe('ParallelOrchestrator', () => { beforeEach(() => { vi.clearAllMocks() + mockCompactSubflowResults.mockImplementation(async (results: unknown) => results) }) it('awaits empty-subflow lifecycle callbacks before returning the empty scope', async () => { @@ -99,9 +109,8 @@ describe('ParallelOrchestrator', () => { const ctx = createContext() const initializePromise = orchestrator.initializeParallelScope(ctx, 'parallel-1') - await Promise.resolve() + await vi.waitFor(() => expect(onBlockStart).toHaveBeenCalledTimes(1)) - expect(onBlockStart).toHaveBeenCalledTimes(1) expect(onBlockComplete).not.toHaveBeenCalled() releaseStart?.() @@ -130,4 +139,171 @@ describe('ParallelOrchestrator', () => { isEmpty: true, }) }) + + it('records resumed later-batch outputs under restored global branch indexes', () => { + const dag = createDag() + dag.nodes.set('task-1', { + id: 'task-1', + block: { + id: 'task-1', + position: { x: 0, y: 0 }, + config: { tool: '', params: {} }, + inputs: {}, + outputs: {}, + metadata: { id: 'function', name: 'Task 1' }, + enabled: true, + }, + incomingEdges: new Set(), + outgoingEdges: new Set(), + metadata: { branchIndex: 0 }, + }) + const orchestrator = new ParallelOrchestrator(dag, createState(), null, {}) + const ctx = createContext({ + parallelBlockMapping: new Map([ + ['task-1', { originalBlockId: 'task', parallelId: 'parallel-1', iterationIndex: 20 }], + ]), + parallelExecutions: new Map([ + [ + 'parallel-1', + { + parallelId: 'parallel-1', + totalBranches: 25, + currentBatchStart: 20, + currentBatchSize: 5, + accumulatedOutputs: new Map([[0, [{ output: 'previous' }]]]), + branchOutputs: new Map(), + }, + ], + ]), + }) + + orchestrator.handleParallelBranchCompletion(ctx, 'parallel-1', 'task-1', { output: 'resumed' }) + + const scope = ctx.parallelExecutions?.get('parallel-1') + expect(scope?.branchOutputs.get(20)).toEqual([{ output: 'resumed' }]) + expect(scope?.branchOutputs.has(0)).toBe(false) + }) + + it('resets only incoming batch branch state when scheduling later batches', async () => { + const dag = createDag() + const incomingBranchId = buildBranchNodeId('task-1', 0) + const previousBranchId = buildBranchNodeId('task-1', 1) + dag.nodes.set(incomingBranchId, { + id: incomingBranchId, + block: { + id: 'task-1', + position: { x: 0, y: 0 }, + config: { tool: '', params: {} }, + inputs: {}, + outputs: {}, + metadata: { id: 'function', name: 'Task 1' }, + enabled: true, + }, + incomingEdges: new Set(), + outgoingEdges: new Set(), + metadata: { parallelId: 'parallel-1', isParallelBranch: true, branchIndex: 0 }, + }) + dag.nodes.set(previousBranchId, { + id: previousBranchId, + block: { + id: 'task-1', + position: { x: 0, y: 0 }, + config: { tool: '', params: {} }, + inputs: {}, + outputs: {}, + metadata: { id: 'function', name: 'Task 1' }, + enabled: true, + }, + incomingEdges: new Set(), + outgoingEdges: new Set(), + metadata: { parallelId: 'parallel-1', isParallelBranch: true, branchIndex: 1 }, + }) + const state = createState() + const orchestrator = new ParallelOrchestrator(dag, state, null, {}) + + await ( + orchestrator as unknown as { + scheduleNextBatch( + ctx: ExecutionContext, + scope: NonNullable extends Map< + string, + infer Scope + > + ? Scope + : never, + nextBatchStart: number + ): Promise + } + ).scheduleNextBatch( + createContext(), + { + parallelId: 'parallel-1', + totalBranches: 3, + batchSize: 1, + currentBatchStart: 0, + currentBatchSize: 2, + accumulatedOutputs: new Map([[1, [{ output: 'previous' }]]]), + branchOutputs: new Map(), + }, + 2 + ) + + expect(state.deleteBlockState).toHaveBeenCalledWith(incomingBranchId) + expect(state.deleteBlockState).not.toHaveBeenCalledWith(previousBranchId) + expect(state.unmarkExecuted).toHaveBeenCalledWith(incomingBranchId) + expect(state.unmarkExecuted).not.toHaveBeenCalledWith(previousBranchId) + }) + + it('compacts accumulated outputs before scheduling later batches', async () => { + const dag = createDag() + const templateBranchId = buildBranchNodeId('task-1', 0) + dag.nodes.set(templateBranchId, { + id: templateBranchId, + block: { + id: 'task-1', + position: { x: 0, y: 0 }, + config: { tool: '', params: {} }, + inputs: {}, + outputs: {}, + metadata: { id: 'function', name: 'Task 1' }, + enabled: true, + }, + incomingEdges: new Set(), + outgoingEdges: new Set(), + metadata: { parallelId: 'parallel-1', isParallelBranch: true, branchIndex: 0 }, + }) + const orchestrator = new ParallelOrchestrator(dag, createState(), null, {}) + const previousOutputs = [{ output: 'previous' }] + const incomingOutputs = [{ output: 'incoming' }] + const compactedPrevious = [{ output: 'compacted-previous' }] + const compactedIncoming = [{ output: 'compacted-incoming' }] + mockCompactSubflowResults.mockResolvedValueOnce([compactedPrevious, compactedIncoming]) + const scope = { + parallelId: 'parallel-1', + totalBranches: 3, + batchSize: 1, + currentBatchStart: 0, + currentBatchSize: 2, + accumulatedOutputs: new Map([[0, previousOutputs]]), + branchOutputs: new Map([[1, incomingOutputs]]), + } + const ctx = createContext({ + parallelExecutions: new Map([['parallel-1', scope]]), + }) + + const result = await orchestrator.aggregateParallelResults(ctx, 'parallel-1') + + expect(result).toMatchObject({ allBranchesComplete: false, completedBranches: 2 }) + expect(mockCompactSubflowResults).toHaveBeenCalledWith( + [previousOutputs, incomingOutputs], + expect.objectContaining({ + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + requireDurable: true, + }) + ) + expect(scope.accumulatedOutputs.get(0)).toBe(compactedPrevious) + expect(scope.accumulatedOutputs.get(1)).toBe(compactedIncoming) + }) }) diff --git a/apps/sim/executor/orchestrators/parallel.ts b/apps/sim/executor/orchestrators/parallel.ts index 7cc10abbee7..aa9d0ad8c6e 100644 --- a/apps/sim/executor/orchestrators/parallel.ts +++ b/apps/sim/executor/orchestrators/parallel.ts @@ -1,24 +1,25 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import { compactSubflowResults } from '@/lib/execution/payloads/serializer' import { DEFAULTS } from '@/executor/constants' import type { DAG } from '@/executor/dag/builder' import type { ParallelScope } from '@/executor/execution/state' import type { BlockStateWriter, ContextExtensions } from '@/executor/execution/types' import type { ExecutionContext, NormalizedBlockOutput } from '@/executor/types' import type { ParallelConfigWithNodes } from '@/executor/types/parallel' -import { ParallelExpander } from '@/executor/utils/parallel-expansion' +import { type ClonedSubflowInfo, ParallelExpander } from '@/executor/utils/parallel-expansion' import { addSubflowErrorLog, emitEmptySubflowEvents, emitSubflowSuccessEvents, extractBranchIndex, - resolveArrayInput, - validateMaxCount, + resolveArrayInputAsync, } from '@/executor/utils/subflow-utils' import type { VariableResolver } from '@/executor/variables/resolver' import type { SerializedParallel } from '@/serializer/types' const logger = createLogger('ParallelOrchestrator') +const DEFAULT_PARALLEL_BATCH_SIZE = 20 export interface ParallelBranchMetadata { branchIndex: number @@ -29,7 +30,7 @@ export interface ParallelBranchMetadata { export interface ParallelAggregationResult { allBranchesComplete: boolean - results?: NormalizedBlockOutput[][] + results?: unknown completedBranches?: number totalBranches?: number } @@ -64,7 +65,7 @@ export class ParallelOrchestrator { let isEmpty = false try { - const resolved = this.resolveBranchCount(ctx, parallelConfig, parallelId) + const resolved = await this.resolveBranchCount(ctx, parallelConfig, parallelId) branchCount = resolved.branchCount items = resolved.items isEmpty = resolved.isEmpty ?? false @@ -81,21 +82,6 @@ export class ParallelOrchestrator { throw new Error(errorMessage) } - const branchError = validateMaxCount( - branchCount, - DEFAULTS.MAX_PARALLEL_BRANCHES, - 'Parallel branch count' - ) - if (branchError) { - logger.error(branchError, { parallelId, branchCount }) - await this.addParallelErrorLog(ctx, parallelId, branchError, { - distribution: parallelConfig.distribution, - branchCount, - }) - this.setErrorScope(ctx, parallelId, branchError) - throw new Error(branchError) - } - if (isEmpty || branchCount === 0) { const scope: ParallelScope = { parallelId, @@ -122,60 +108,27 @@ export class ParallelOrchestrator { return scope } - const { entryNodes, clonedSubflows } = this.expander.expandParallel( + const batchSize = this.resolveBatchSize(parallelConfig.batchSize) + const currentBatchSize = Math.min(batchSize, branchCount) + const batchItems = items?.slice(0, currentBatchSize) + const { entryNodes, clonedSubflows, allBranchNodes } = this.expander.expandParallel( this.dag, parallelId, - branchCount, - items + currentBatchSize, + batchItems, + { branchIndexOffset: 0, totalBranches: branchCount } ) - // Register cloned subflows in the parent map so iteration context resolves correctly. - // Build a per-branch clone map so nested clones point to the cloned parent, not the original. - if (clonedSubflows.length > 0 && ctx.subflowParentMap) { - const branchCloneMaps = new Map>() - for (const clone of clonedSubflows) { - let map = branchCloneMaps.get(clone.outerBranchIndex) - if (!map) { - map = new Map() - branchCloneMaps.set(clone.outerBranchIndex, map) - } - map.set(clone.originalId, clone.clonedId) - } - - for (const clone of clonedSubflows) { - const originalEntry = ctx.subflowParentMap.get(clone.originalId) - if (originalEntry) { - const cloneMap = branchCloneMaps.get(clone.outerBranchIndex) - const clonedParentId = cloneMap?.get(originalEntry.parentId) - if (clonedParentId) { - // Parent was also cloned — this is the original (branch 0) inside the cloned parent - ctx.subflowParentMap.set(clone.clonedId, { - parentId: clonedParentId, - parentType: originalEntry.parentType, - branchIndex: 0, - }) - } else { - // Parent was not cloned — direct child of the expanding parallel - ctx.subflowParentMap.set(clone.clonedId, { - parentId: parallelId, - parentType: 'parallel', - branchIndex: clone.outerBranchIndex, - }) - } - } else { - // Not in parent map — direct child of the expanding parallel - ctx.subflowParentMap.set(clone.clonedId, { - parentId: parallelId, - parentType: 'parallel', - branchIndex: clone.outerBranchIndex, - }) - } - } - } + this.registerClonedSubflows(ctx, parallelId, clonedSubflows) + this.registerBranchMappings(ctx, parallelId, allBranchNodes) const scope: ParallelScope = { parallelId, totalBranches: branchCount, + batchSize, + currentBatchStart: 0, + currentBatchSize, + accumulatedOutputs: new Map(), branchOutputs: new Map(), items, } @@ -196,6 +149,8 @@ export class ParallelOrchestrator { logger.info('Parallel scope initialized', { parallelId, branchCount, + batchSize, + currentBatchSize, entryNodeCount: entryNodes.length, newEntryNodes: newEntryNodes.length, }) @@ -203,16 +158,16 @@ export class ParallelOrchestrator { return scope } - private resolveBranchCount( + private async resolveBranchCount( ctx: ExecutionContext, config: SerializedParallel, parallelId: string - ): { branchCount: number; items?: any[]; isEmpty?: boolean } { + ): Promise<{ branchCount: number; items?: any[]; isEmpty?: boolean }> { if (config.parallelType === 'count') { return { branchCount: config.count ?? 1 } } - const items = this.resolveDistributionItems(ctx, config) + const items = await this.resolveDistributionItems(ctx, config) if (items.length === 0) { logger.info('Parallel has empty distribution, skipping parallel body', { parallelId }) return { branchCount: 0, items: [], isEmpty: true } @@ -251,7 +206,10 @@ export class ParallelOrchestrator { ctx.parallelExecutions.set(parallelId, scope) } - private resolveDistributionItems(ctx: ExecutionContext, config: SerializedParallel): any[] { + private async resolveDistributionItems( + ctx: ExecutionContext, + config: SerializedParallel + ): Promise { if ( config.distribution === undefined || config.distribution === null || @@ -261,7 +219,63 @@ export class ParallelOrchestrator { 'Parallel collection distribution is empty. Provide an array or a reference that resolves to a collection.' ) } - return resolveArrayInput(ctx, config.distribution, this.resolver) + return resolveArrayInputAsync(ctx, config.distribution, this.resolver) + } + + private resolveBatchSize(batchSize: unknown): number { + const parsed = + typeof batchSize === 'number' ? batchSize : Number.parseInt(String(batchSize), 10) + if (Number.isNaN(parsed)) { + return DEFAULT_PARALLEL_BATCH_SIZE + } + return Math.max(1, Math.min(DEFAULTS.MAX_PARALLEL_BRANCHES, parsed)) + } + + private registerClonedSubflows( + ctx: ExecutionContext, + parallelId: string, + clonedSubflows: ClonedSubflowInfo[] + ): void { + if (clonedSubflows.length === 0 || !ctx.subflowParentMap) { + return + } + + const branchCloneMaps = new Map>() + for (const clone of clonedSubflows) { + let map = branchCloneMaps.get(clone.outerBranchIndex) + if (!map) { + map = new Map() + branchCloneMaps.set(clone.outerBranchIndex, map) + } + map.set(clone.originalId, clone.clonedId) + } + + for (const clone of clonedSubflows) { + const originalEntry = ctx.subflowParentMap.get(clone.originalId) + if (originalEntry) { + const cloneMap = branchCloneMaps.get(clone.outerBranchIndex) + const clonedParentId = cloneMap?.get(originalEntry.parentId) + if (clonedParentId) { + ctx.subflowParentMap.set(clone.clonedId, { + parentId: clonedParentId, + parentType: originalEntry.parentType, + branchIndex: 0, + }) + } else { + ctx.subflowParentMap.set(clone.clonedId, { + parentId: parallelId, + parentType: 'parallel', + branchIndex: clone.outerBranchIndex, + }) + } + } else { + ctx.subflowParentMap.set(clone.clonedId, { + parentId: parallelId, + parentType: 'parallel', + branchIndex: clone.outerBranchIndex, + }) + } + } } /** @@ -282,7 +296,11 @@ export class ParallelOrchestrator { return } - const branchIndex = extractBranchIndex(nodeId) + const mappedBranch = ctx.parallelBlockMapping?.get(nodeId) + const branchIndex = + mappedBranch?.parallelId === parallelId + ? mappedBranch.iterationIndex + : (this.dag.nodes.get(nodeId)?.metadata.branchIndex ?? extractBranchIndex(nodeId)) if (branchIndex === null) { logger.warn('Could not extract branch index from node ID', { nodeId }) return @@ -304,33 +322,162 @@ export class ParallelOrchestrator { return { allBranchesComplete: false } } + const accumulatedOutputs = + scope.accumulatedOutputs ?? new Map() + for (const [branchIndex, outputs] of scope.branchOutputs.entries()) { + accumulatedOutputs.set(branchIndex, outputs) + } + scope.accumulatedOutputs = accumulatedOutputs + scope.branchOutputs = new Map() + + const nextBatchStart = + (scope.currentBatchStart ?? 0) + (scope.currentBatchSize ?? scope.totalBranches) + if (nextBatchStart < scope.totalBranches) { + /** + * Compact accumulated outputs before scheduling the next batch. Each + * block output is already individually compacted by `block-executor`, but + * many below-threshold branch results can still exceed the aggregate + * threshold over time. Re-running the existing subflow compactor over the + * accumulated entries forces aggregate-size spills while existing + * LargeValueRefs stay stable. + */ + if (accumulatedOutputs.size > 0) { + const accumulatedBranchIndexes = Array.from(accumulatedOutputs.keys()).sort((a, b) => a - b) + const accumulatedResults = accumulatedBranchIndexes.map( + (idx) => accumulatedOutputs.get(idx) ?? [] + ) + const compactedAccumulated = await compactSubflowResults(accumulatedResults, { + workspaceId: ctx.workspaceId, + workflowId: ctx.workflowId, + executionId: ctx.executionId, + userId: ctx.userId, + requireDurable: true, + }) + accumulatedBranchIndexes.forEach((branchIdx, position) => { + accumulatedOutputs.set(branchIdx, compactedAccumulated[position]) + }) + } + await this.scheduleNextBatch(ctx, scope, nextBatchStart) + return { + allBranchesComplete: false, + completedBranches: accumulatedOutputs.size, + totalBranches: scope.totalBranches, + } + } + const results: NormalizedBlockOutput[][] = [] for (let i = 0; i < scope.totalBranches; i++) { - const branchOutputs = scope.branchOutputs.get(i) + const branchOutputs = accumulatedOutputs.get(i) if (!branchOutputs) { logger.warn('Missing branch output during parallel aggregation', { parallelId, branch: i }) } results.push(branchOutputs ?? []) } - const output = { results } + const compactedResults = await compactSubflowResults(results, { + workspaceId: ctx.workspaceId, + workflowId: ctx.workflowId, + executionId: ctx.executionId, + userId: ctx.userId, + requireDurable: true, + }) + const output = { results: compactedResults } this.state.setBlockOutput(parallelId, output) + scope.accumulatedOutputs = new Map() await emitSubflowSuccessEvents(ctx, parallelId, 'parallel', output, this.contextExtensions) return { allBranchesComplete: true, - results, + results: output.results, completedBranches: scope.totalBranches, totalBranches: scope.totalBranches, } } + + private async scheduleNextBatch( + ctx: ExecutionContext, + scope: ParallelScope, + nextBatchStart: number + ): Promise { + const batchSize = scope.batchSize ?? DEFAULT_PARALLEL_BATCH_SIZE + const remaining = scope.totalBranches - nextBatchStart + const currentBatchSize = Math.min(batchSize, remaining) + const batchItems = scope.items?.slice(nextBatchStart, nextBatchStart + currentBatchSize) + + const { entryNodes, clonedSubflows, allBranchNodes } = this.expander.expandParallel( + this.dag, + scope.parallelId, + currentBatchSize, + batchItems, + { branchIndexOffset: nextBatchStart, totalBranches: scope.totalBranches } + ) + + this.registerClonedSubflows(ctx, scope.parallelId, clonedSubflows) + this.registerBranchMappings(ctx, scope.parallelId, allBranchNodes) + this.resetBatchExecutionState(allBranchNodes) + + scope.currentBatchStart = nextBatchStart + scope.currentBatchSize = currentBatchSize + + if (!ctx.pendingDynamicNodes) { + ctx.pendingDynamicNodes = [] + } + ctx.pendingDynamicNodes.push(...entryNodes) + + logger.info('Scheduled next parallel batch', { + parallelId: scope.parallelId, + nextBatchStart, + currentBatchSize, + totalBranches: scope.totalBranches, + }) + } + + private resetBatchExecutionState(branchNodeIds: string[]): void { + for (const nodeId of branchNodeIds) { + const node = this.dag.nodes.get(nodeId) + if (!node?.metadata.isParallelBranch) { + continue + } + this.state.unmarkExecuted(nodeId) + this.state.deleteBlockState(nodeId) + } + } + + private registerBranchMappings( + ctx: ExecutionContext, + parallelId: string, + branchNodeIds: string[] + ): void { + if (branchNodeIds.length === 0) { + return + } + + if (!ctx.parallelBlockMapping) { + ctx.parallelBlockMapping = new Map() + } + + for (const nodeId of branchNodeIds) { + const node = this.dag.nodes.get(nodeId) + const branchIndex = node?.metadata.branchIndex ?? extractBranchIndex(nodeId) + if (branchIndex === null || branchIndex === undefined) { + continue + } + + ctx.parallelBlockMapping.set(nodeId, { + originalBlockId: node?.metadata.originalBlockId ?? nodeId, + parallelId, + iterationIndex: branchIndex, + }) + } + } + extractBranchMetadata(nodeId: string): ParallelBranchMetadata | null { const node = this.dag.nodes.get(nodeId) if (!node?.metadata.isParallelBranch) { return null } - const branchIndex = extractBranchIndex(nodeId) + const branchIndex = node.metadata.branchIndex ?? extractBranchIndex(nodeId) if (branchIndex === null) { return null } diff --git a/apps/sim/executor/types.ts b/apps/sim/executor/types.ts index 2d48bb5a98f..d2569706085 100644 --- a/apps/sim/executor/types.ts +++ b/apps/sim/executor/types.ts @@ -290,6 +290,8 @@ export interface ExecutionContext { workflowId: string workspaceId?: string executionId?: string + largeValueExecutionIds?: string[] + allowLargeValueWorkflowScope?: boolean userId?: string isDeployedContext?: boolean enforceCredentialAccess?: boolean @@ -344,6 +346,10 @@ export interface ExecutionContext { { parallelId: string totalBranches: number + batchSize?: number + currentBatchStart?: number + currentBatchSize?: number + accumulatedOutputs?: Map branchOutputs: Map parallelType?: 'count' | 'collection' items?: any[] diff --git a/apps/sim/executor/utils/block-reference.ts b/apps/sim/executor/utils/block-reference.ts index edf909a6d3b..082a9339782 100644 --- a/apps/sim/executor/utils/block-reference.ts +++ b/apps/sim/executor/utils/block-reference.ts @@ -1,6 +1,10 @@ import { USER_FILE_ACCESSIBLE_PROPERTIES } from '@/lib/workflows/types' import { normalizeName } from '@/executor/constants' -import { navigatePath } from '@/executor/variables/resolvers/reference' +import { + type AsyncPathNavigator, + navigatePath, + type ResolutionContext, +} from '@/executor/variables/resolvers/reference' /** * A single schema node encountered while walking an `OutputSchema`. Captures @@ -204,7 +208,11 @@ function getSchemaFieldNames(schema: OutputSchema | undefined): string[] { export function resolveBlockReference( blockName: string, pathParts: string[], - context: BlockReferenceContext + context: BlockReferenceContext, + options: { + allowLargeValueRefs?: boolean + executionContext?: ResolutionContext['executionContext'] + } = {} ): BlockReferenceResult | undefined { const normalizedName = normalizeName(blockName) const blockId = context.blockNameMapping[normalizedName] @@ -227,7 +235,42 @@ export function resolveBlockReference( return { value: blockOutput, blockId } } - const value = navigatePath(blockOutput, pathParts) + const value = navigatePath(blockOutput, pathParts, options) + + const schema = context.blockOutputSchemas?.[blockId] + if (value === undefined && schema) { + if (!isPathInSchema(schema, pathParts)) { + throw new InvalidFieldError(blockName, pathParts.join('.'), getSchemaFieldNames(schema)) + } + } + + return { value, blockId } +} + +export async function resolveBlockReferenceAsync( + blockName: string, + pathParts: string[], + context: BlockReferenceContext, + resolutionContext: ResolutionContext, + navigatePathAsync: AsyncPathNavigator +): Promise { + const normalizedName = normalizeName(blockName) + const blockId = context.blockNameMapping[normalizedName] + + if (!blockId) { + return undefined + } + + const blockOutput = context.blockData[blockId] + if (blockOutput === undefined) { + return { value: undefined, blockId } + } + + if (pathParts.length === 0) { + return { value: blockOutput, blockId } + } + + const value = await navigatePathAsync(blockOutput, pathParts, resolutionContext) const schema = context.blockOutputSchemas?.[blockId] if (value === undefined && schema) { diff --git a/apps/sim/executor/utils/output-filter.ts b/apps/sim/executor/utils/output-filter.ts index 5da00faba53..95c3cab5397 100644 --- a/apps/sim/executor/utils/output-filter.ts +++ b/apps/sim/executor/utils/output-filter.ts @@ -1,3 +1,4 @@ +import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref' import { filterHiddenOutputKeys } from '@/lib/logs/execution/trace-spans/trace-spans' import { getBlock } from '@/blocks' import { isHiddenFromDisplay } from '@/blocks/types' @@ -27,6 +28,9 @@ export function filterOutputForLog( if (typeof output !== 'object' || output === null || Array.isArray(output)) { return output as NormalizedBlockOutput } + if (isLargeValueRef(output)) { + return output as NormalizedBlockOutput + } const blockConfig = blockType ? getBlock(blockType) : undefined const filtered: NormalizedBlockOutput = {} const additionalHiddenKeys = options?.additionalHiddenKeys ?? [] diff --git a/apps/sim/executor/utils/parallel-expansion.test.ts b/apps/sim/executor/utils/parallel-expansion.test.ts index bcb2fbeb5c5..67f0e865aef 100644 --- a/apps/sim/executor/utils/parallel-expansion.test.ts +++ b/apps/sim/executor/utils/parallel-expansion.test.ts @@ -207,6 +207,65 @@ describe('Nested parallel expansion + edge resolution', () => { expect(readyAfterClonedInnerEnd).toContain(outerEndId) }) + it('uses global branch indexes for nested subflow clones in later batches', () => { + const outerParallelId = 'outer-parallel' + const innerParallelId = 'inner-parallel' + const functionId = 'func-1' + + const workflow: SerializedWorkflow = { + version: '1', + blocks: [ + createBlock('start', BlockType.STARTER), + createBlock(outerParallelId, BlockType.PARALLEL), + createBlock(innerParallelId, BlockType.PARALLEL), + createBlock(functionId, BlockType.FUNCTION), + ], + connections: [ + { source: 'start', target: outerParallelId }, + { + source: outerParallelId, + target: innerParallelId, + sourceHandle: 'parallel-start-source', + }, + { + source: innerParallelId, + target: functionId, + sourceHandle: 'parallel-start-source', + }, + ], + loops: {}, + parallels: { + [innerParallelId]: { + id: innerParallelId, + nodes: [functionId], + count: 1, + parallelType: 'count', + }, + [outerParallelId]: { + id: outerParallelId, + nodes: [innerParallelId], + count: 4, + parallelType: 'count', + }, + }, + } + + const builder = new DAGBuilder() + const dag = builder.build(workflow) + const expander = new ParallelExpander() + const result = expander.expandParallel(dag, outerParallelId, 2, undefined, { + branchIndexOffset: 2, + totalBranches: 4, + }) + + expect(result.entryNodes).not.toContain(buildParallelSentinelStartId(innerParallelId)) + expect(result.clonedSubflows.map((clone) => clone.outerBranchIndex)).toEqual([2, 3]) + expect(result.clonedSubflows.map((clone) => clone.clonedId)).toEqual([ + `${innerParallelId}__obranch-2`, + `${innerParallelId}__obranch-3`, + ]) + }) + it('3-level nesting: pre-expansion clone IDs do not collide with runtime expansion', () => { const p1 = 'p1' const p2 = 'p2' @@ -251,7 +310,7 @@ describe('Nested parallel expansion + edge resolution', () => { // P3 should also be cloned (inside P2__obranch-1) with a __clone prefix const p3Clone = p1Result.clonedSubflows.find((c) => c.originalId === p3)! expect(p3Clone).toBeDefined() - expect(p3Clone.clonedId).toMatch(/^p3__clone\d+__obranch-1$/) + expect(p3Clone.clonedId).toMatch(/^p3__clone[0-9a-f]{24}__obranch-1$/) expect(stripCloneSuffixes(p3Clone.clonedId)).toBe('p3') // Step 2: Expand P2 (original, branch 0 of P1) — this creates P3__obranch-1 at runtime diff --git a/apps/sim/executor/utils/parallel-expansion.ts b/apps/sim/executor/utils/parallel-expansion.ts index 6d59af91c8d..f98c9f49e57 100644 --- a/apps/sim/executor/utils/parallel-expansion.ts +++ b/apps/sim/executor/utils/parallel-expansion.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { sha256Hex } from '@sim/security/hash' import { EDGE } from '@/executor/constants' import type { DAG, DAGNode } from '@/executor/dag/builder' import type { SerializedBlock } from '@/serializer/types' @@ -29,14 +30,12 @@ export interface ExpansionResult { } export class ParallelExpander { - /** Monotonically increasing counter for generating unique pre-expansion clone IDs. */ - private cloneSeq = 0 - expandParallel( dag: DAG, parallelId: string, branchCount: number, - distributionItems?: any[] + distributionItems?: any[], + options: { branchIndexOffset?: number; totalBranches?: number } = {} ): ExpansionResult { const config = dag.parallelConfigs.get(parallelId) if (!config) { @@ -64,6 +63,8 @@ export class ParallelExpander { const regularSet = new Set(regularBlocks) const allBranchNodes: string[] = [] + const branchIndexOffset = options.branchIndexOffset ?? 0 + const branchTotal = options.totalBranches ?? branchCount for (const blockId of regularBlocks) { const templateId = buildBranchNodeId(blockId, 0) @@ -76,10 +77,16 @@ export class ParallelExpander { for (let i = 0; i < branchCount; i++) { const branchNodeId = buildBranchNodeId(blockId, i) + const globalBranchIndex = branchIndexOffset + i allBranchNodes.push(branchNodeId) if (i === 0) { - this.updateBranchMetadata(templateNode, i, branchCount, distributionItems?.[i]) + this.updateBranchMetadata( + templateNode, + globalBranchIndex, + branchTotal, + distributionItems?.[i] + ) continue } @@ -87,7 +94,8 @@ export class ParallelExpander { templateNode, blockId, i, - branchCount, + globalBranchIndex, + branchTotal, distributionItems?.[i] ) dag.nodes.set(branchNodeId, branchNode) @@ -114,20 +122,22 @@ export class ParallelExpander { ? buildParallelSentinelEndId(subflowId) : buildSentinelEndId(subflowId) - // Branch 0 uses original nodes - if (dag.nodes.has(startId)) entryNodes.push(startId) - if (dag.nodes.has(endId)) terminalNodes.push(endId) + for (let i = 0; i < branchCount; i++) { + const globalBranchIndex = branchIndexOffset + i + if (globalBranchIndex === 0) { + if (dag.nodes.has(startId)) entryNodes.push(startId) + if (dag.nodes.has(endId)) terminalNodes.push(endId) + continue + } - // Branches 1..N clone the entire subflow graph (recursively for deep nesting) - for (let i = 1; i < branchCount; i++) { - const cloned = this.cloneNestedSubflow(dag, subflowId, i, clonedSubflows) + const cloned = this.cloneNestedSubflow(dag, subflowId, globalBranchIndex, clonedSubflows) entryNodes.push(cloned.startId) terminalNodes.push(cloned.endId) clonedSubflows.push({ clonedId: cloned.clonedId, originalId: subflowId, - outerBranchIndex: i, + outerBranchIndex: globalBranchIndex, }) } } @@ -161,11 +171,12 @@ export class ParallelExpander { private cloneTemplateNode( template: DAGNode, originalBlockId: string, + localBranchIndex: number, branchIndex: number, branchTotal: number, distributionItem?: any ): DAGNode { - const branchNodeId = buildBranchNodeId(originalBlockId, branchIndex) + const branchNodeId = buildBranchNodeId(originalBlockId, localBranchIndex) const blockClone: SerializedBlock = { ...template.block, id: branchNodeId, @@ -201,7 +212,11 @@ export class ParallelExpander { const baseTargetId = extractBaseBlockId(edge.target) if (!blocksSet.has(baseTargetId)) continue - for (let i = 1; i < branchCount; i++) { + // Include branch 0 so per-batch re-expansion restores the template's + // incoming-edge bookkeeping that earlier batches consumed during + // edge processing. Without this, identifyBoundaryNodes mis-classifies + // chained children as entry nodes after the first batch. + for (let i = 0; i < branchCount; i++) { const sourceNodeId = buildBranchNodeId(blockId, i) const targetNodeId = buildBranchNodeId(baseTargetId, i) const sourceNode = dag.nodes.get(sourceNodeId) @@ -278,14 +293,20 @@ export class ParallelExpander { /** * Generates a unique clone ID for pre-expansion cloning. * - * Pre-expansion clones use `{originalId}__clone{N}__obranch-{branchIndex}` instead + * Pre-expansion clones use `{originalId}__clone{digest}__obranch-{branchIndex}` instead * of the plain `{originalId}__obranch-{branchIndex}` used by runtime expansion. - * The `__clone{N}` segment (from a monotonic counter) prevents naming collisions - * when the original (branch-0) subflow later expands at runtime and creates - * `{child}__obranch-{branchIndex}`. + * The clone segment prevents naming collisions when the original (branch-0) + * subflow later expands at runtime and creates `{child}__obranch-{branchIndex}`. + * Keeping it deterministic lets pause/resume rebuild the same active branch IDs. */ - private buildPreCloneId(originalId: string, outerBranchIndex: number): string { - return `${originalId}__clone${this.cloneSeq++}__obranch-${outerBranchIndex}` + private buildPreCloneIdForParent( + originalId: string, + outerBranchIndex: number, + parentCloneId: string + ): string { + const input = `${parentCloneId}:${originalId}:${outerBranchIndex}` + const digest = sha256Hex(input).slice(0, 24) + return `${originalId}__clone${digest}__obranch-${outerBranchIndex}` } /** @@ -293,8 +314,8 @@ export class ParallelExpander { * * The top-level subflow gets a standard `__obranch-{N}` clone ID (needed by * `findEffectiveContainerId` at runtime). All deeper children — both containers - * and regular blocks — receive unique `__clone{N}__obranch-{M}` IDs via - * {@link buildPreCloneId} to avoid collisions with runtime expansion. + * and regular blocks — receive deterministic `__clone{N}__obranch-{M}` IDs to + * avoid collisions with runtime expansion. */ private cloneNestedSubflow( dag: DAG, @@ -357,7 +378,7 @@ export class ParallelExpander { const isNestedLoop = dag.loopConfigs.has(blockId) if (isNestedParallel || isNestedLoop) { - const nestedClonedId = this.buildPreCloneId(blockId, outerBranchIndex) + const nestedClonedId = this.buildPreCloneIdForParent(blockId, outerBranchIndex, clonedId) clonedBlockIds.push(nestedClonedId) const innerResult = this.cloneSubflowGraph( @@ -377,7 +398,7 @@ export class ParallelExpander { outerBranchIndex, }) } else { - const clonedBlockId = this.buildPreCloneId(blockId, outerBranchIndex) + const clonedBlockId = this.buildPreCloneIdForParent(blockId, outerBranchIndex, clonedId) clonedBlockIds.push(clonedBlockId) if (isParallel) { diff --git a/apps/sim/executor/utils/subflow-utils.test.ts b/apps/sim/executor/utils/subflow-utils.test.ts index 18f7e2097d8..478319d6ca9 100644 --- a/apps/sim/executor/utils/subflow-utils.test.ts +++ b/apps/sim/executor/utils/subflow-utils.test.ts @@ -4,83 +4,99 @@ import { describe, expect, it, vi } from 'vitest' import type { ExecutionContext } from '@/executor/types' import type { VariableResolver } from '@/executor/variables/resolver' -import { resolveArrayInput } from './subflow-utils' +import { findEffectiveContainerId, resolveArrayInputAsync } from './subflow-utils' -describe('resolveArrayInput', () => { +describe('resolveArrayInputAsync', () => { const fakeCtx = {} as unknown as ExecutionContext - it('returns arrays as-is', () => { - expect(resolveArrayInput(fakeCtx, [1, 2, 3], null)).toEqual([1, 2, 3]) + it('returns arrays as-is', async () => { + await expect(resolveArrayInputAsync(fakeCtx, [1, 2, 3], null)).resolves.toEqual([1, 2, 3]) }) - it('converts plain objects to entries', () => { - expect(resolveArrayInput(fakeCtx, { a: 1, b: 2 }, null)).toEqual([ + it('converts plain objects to entries', async () => { + await expect(resolveArrayInputAsync(fakeCtx, { a: 1, b: 2 }, null)).resolves.toEqual([ ['a', 1], ['b', 2], ]) }) - it('returns empty array when a pure reference resolves to null (skipped block)', () => { + it('returns empty array when a pure reference resolves to null (skipped block)', async () => { // `resolveSingleReference` returns `null` for a reference that points at a // block that exists in the workflow but did not execute on this path. // A loop/parallel over such a reference should run zero iterations rather // than fail the workflow. const resolver = { - resolveSingleReference: vi.fn().mockReturnValue(null), + resolveSingleReference: vi.fn().mockResolvedValue(null), } as unknown as VariableResolver - const result = resolveArrayInput(fakeCtx, '', resolver) + const result = await resolveArrayInputAsync(fakeCtx, '', resolver) expect(result).toEqual([]) expect(resolver.resolveSingleReference).toHaveBeenCalled() }) - it('returns the array from a pure reference that resolved to an array', () => { + it('returns the array from a pure reference that resolved to an array', async () => { const resolver = { - resolveSingleReference: vi.fn().mockReturnValue([1, 2, 3]), + resolveSingleReference: vi.fn().mockResolvedValue([1, 2, 3]), } as unknown as VariableResolver - expect(resolveArrayInput(fakeCtx, '', resolver)).toEqual([1, 2, 3]) + await expect(resolveArrayInputAsync(fakeCtx, '', resolver)).resolves.toEqual([ + 1, 2, 3, + ]) }) - it('converts resolved objects to entries', () => { + it('converts resolved objects to entries', async () => { const resolver = { - resolveSingleReference: vi.fn().mockReturnValue({ x: 1, y: 2 }), + resolveSingleReference: vi.fn().mockResolvedValue({ x: 1, y: 2 }), } as unknown as VariableResolver - expect(resolveArrayInput(fakeCtx, '', resolver)).toEqual([ + await expect(resolveArrayInputAsync(fakeCtx, '', resolver)).resolves.toEqual([ ['x', 1], ['y', 2], ]) }) - it('throws when a pure reference resolves to a non-array, non-object, non-null value', () => { + it('throws when a pure reference resolves to a non-array, non-object, non-null value', async () => { const resolver = { - resolveSingleReference: vi.fn().mockReturnValue(42), + resolveSingleReference: vi.fn().mockResolvedValue(42), } as unknown as VariableResolver - expect(() => resolveArrayInput(fakeCtx, '', resolver)).toThrow( + await expect(resolveArrayInputAsync(fakeCtx, '', resolver)).rejects.toThrow( /did not resolve to an array or object/ ) }) - it('throws when a pure reference resolves to undefined (unknown block)', () => { + it('throws when a pure reference resolves to undefined (unknown block)', async () => { // `undefined` means the reference could not be matched to any block at // all (typo / deleted block). This must still fail loudly. const resolver = { - resolveSingleReference: vi.fn().mockReturnValue(undefined), + resolveSingleReference: vi.fn().mockResolvedValue(undefined), } as unknown as VariableResolver - expect(() => resolveArrayInput(fakeCtx, '', resolver)).toThrow( + await expect(resolveArrayInputAsync(fakeCtx, '', resolver)).rejects.toThrow( /did not resolve to an array or object/ ) }) - it('parses a JSON array string', () => { - expect(resolveArrayInput(fakeCtx, '[1, 2, 3]', null)).toEqual([1, 2, 3]) + it('parses a JSON array string', async () => { + await expect(resolveArrayInputAsync(fakeCtx, '[1, 2, 3]', null)).resolves.toEqual([1, 2, 3]) + }) + + it('throws on a string that is neither a reference nor valid JSON array/object', async () => { + await expect(resolveArrayInputAsync(fakeCtx, 'not json', null)).rejects.toThrow() }) +}) + +describe('findEffectiveContainerId', () => { + it('finds pre-cloned nested subflow IDs with clone sequence suffixes', () => { + const executionMap = new Map([ + ['inner-parallel', {}], + ['inner-parallel__obranch-2', {}], + ['inner-parallel__clone3__obranch-2', {}], + ]) - it('throws on a string that is neither a reference nor valid JSON array/object', () => { - expect(() => resolveArrayInput(fakeCtx, 'not json', null)).toThrow() + expect( + findEffectiveContainerId('inner-parallel', 'leaf__clone7__obranch-2₍0₎', executionMap) + ).toBe('inner-parallel__clone3__obranch-2') }) }) diff --git a/apps/sim/executor/utils/subflow-utils.ts b/apps/sim/executor/utils/subflow-utils.ts index 01765360787..7f363365627 100644 --- a/apps/sim/executor/utils/subflow-utils.ts +++ b/apps/sim/executor/utils/subflow-utils.ts @@ -96,7 +96,7 @@ export function isBranchNodeId(nodeId: string): boolean { const OUTER_BRANCH_PATTERN = /__obranch-(\d+)/ const OUTER_BRANCH_STRIP_PATTERN = /__obranch-\d+/g -const CLONE_SEQ_STRIP_PATTERN = /__clone\d+/g +const CLONE_DIGEST_STRIP_PATTERN = /__clone[0-9a-f]+/gi /** * Extracts the outer branch index from a cloned subflow ID. @@ -114,7 +114,7 @@ export function extractOuterBranchIndex(clonedId: string): number | undefined { */ export function stripCloneSuffixes(nodeId: string): string { return extractBaseBlockId( - nodeId.replace(OUTER_BRANCH_STRIP_PATTERN, '').replace(CLONE_SEQ_STRIP_PATTERN, '') + nodeId.replace(OUTER_BRANCH_STRIP_PATTERN, '').replace(CLONE_DIGEST_STRIP_PATTERN, '') ) } @@ -130,7 +130,7 @@ export function buildClonedSubflowId(originalId: string, branchIndex: number): s * returning the original workflow-level subflow ID. */ export function stripOuterBranchSuffix(id: string): string { - return id.replace(OUTER_BRANCH_STRIP_PATTERN, '').replace(CLONE_SEQ_STRIP_PATTERN, '') + return id.replace(OUTER_BRANCH_STRIP_PATTERN, '').replace(CLONE_DIGEST_STRIP_PATTERN, '') } /** @@ -154,10 +154,30 @@ export function findEffectiveContainerId( // and cloned variants coexist in the map; the clone is the correct scope. const match = currentNodeId.match(OUTER_BRANCH_PATTERN) if (match) { - const candidateId = buildClonedSubflowId(originalId, Number.parseInt(match[1], 10)) + const branchIndex = Number.parseInt(match[1], 10) + const cloneSuffix = `__obranch-${branchIndex}` + if (currentNodeId.includes('__clone')) { + for (const scopeId of executionMap.keys()) { + if ( + scopeId.includes('__clone') && + scopeId.endsWith(cloneSuffix) && + stripOuterBranchSuffix(scopeId) === originalId + ) { + return scopeId + } + } + } + + const candidateId = buildClonedSubflowId(originalId, branchIndex) if (executionMap.has(candidateId)) { return candidateId } + + for (const scopeId of executionMap.keys()) { + if (scopeId.endsWith(cloneSuffix) && stripOuterBranchSuffix(scopeId) === originalId) { + return scopeId + } + } } // Return original ID — for branch-0 (non-cloned) or when scope is missing. @@ -179,26 +199,14 @@ export function normalizeNodeId(nodeId: string): string { } /** - * Validates that a count doesn't exceed a maximum limit. - * Returns an error message if validation fails, undefined otherwise. - */ -export function validateMaxCount(count: number, max: number, itemType: string): string | undefined { - if (count > max) { - return `${itemType} (${count}) exceeds maximum allowed (${max}). Execution blocked.` - } - return undefined -} - -/** - * Resolves array input at runtime. Handles arrays, objects, references, and JSON strings. - * Used by both loop forEach and parallel distribution resolution. - * Throws an error if resolution fails. + * Async variant used by execution paths that may need durable large-value or + * explicit UserFile.base64 materialization while resolving collection inputs. */ -export function resolveArrayInput( +export async function resolveArrayInputAsync( ctx: ExecutionContext, items: any, resolver: VariableResolver | null -): any[] { +): Promise { if (Array.isArray(items)) { return items } @@ -210,7 +218,7 @@ export function resolveArrayInput( if (typeof items === 'string') { if (items.startsWith(REFERENCE.START) && items.endsWith(REFERENCE.END) && resolver) { try { - const resolved = resolver.resolveSingleReference(ctx, '', items) + const resolved = await resolver.resolveSingleReference(ctx, '', items) if (Array.isArray(resolved)) { return resolved } @@ -249,7 +257,7 @@ export function resolveArrayInput( if (resolver) { try { - const resolved = resolver.resolveInputs(ctx, 'subflow_items', { items }).items + const resolved = (await resolver.resolveInputs(ctx, 'subflow_items', { items })).items if (Array.isArray(resolved)) { return resolved } @@ -408,7 +416,7 @@ export async function emitSubflowSuccessEvents( ctx: ExecutionContext, blockId: string, blockType: 'loop' | 'parallel', - output: { results: any[] }, + output: { results: unknown }, contextExtensions: ContextExtensions | null ): Promise { const now = new Date().toISOString() diff --git a/apps/sim/executor/variables/resolver.test.ts b/apps/sim/executor/variables/resolver.test.ts index 9fe0e6273fd..6058a2053e8 100644 --- a/apps/sim/executor/variables/resolver.test.ts +++ b/apps/sim/executor/variables/resolver.test.ts @@ -18,6 +18,7 @@ function createBlock(id: string, name: string, type: string, params = {}): Seria outputs: { result: 'string', items: 'json', + file: 'file', }, enabled: true, } @@ -39,6 +40,16 @@ function createResolver(language = 'javascript') { state.setBlockOutput('producer', { result: 'hello world', items: ['a', 'b'], + file: { + id: 'file-1', + name: 'image.png', + url: 'https://example.com/image.png', + key: 'execution/workspace-1/workflow-1/execution-1/image.png', + context: 'execution', + size: 12 * 1024 * 1024, + type: 'image/png', + base64: 'large-inline-base64', + }, }) const ctx = { blockStates: state.getBlockStates(), @@ -61,18 +72,18 @@ function createResolver(language = 'javascript') { } describe('VariableResolver function block inputs', () => { - it('returns empty inputs when params are missing', () => { + it('returns empty inputs when params are missing', async () => { const { block, ctx, resolver } = createResolver() - const result = resolver.resolveInputsForFunctionBlock(ctx, 'function', undefined, block) + const result = await resolver.resolveInputsForFunctionBlock(ctx, 'function', undefined, block) expect(result).toEqual({ resolvedInputs: {}, displayInputs: {}, contextVariables: {} }) }) - it('resolves JavaScript block references through globalThis context variables', () => { + it('resolves JavaScript block references through globalThis context variables', async () => { const { block, ctx, resolver } = createResolver('javascript') - const result = resolver.resolveInputsForFunctionBlock( + const result = await resolver.resolveInputsForFunctionBlock( ctx, 'function', { code: 'return ' }, @@ -84,10 +95,518 @@ describe('VariableResolver function block inputs', () => { expect(result.contextVariables).toEqual({ __blockRef_0: 'hello world' }) }) - it('resolves Python block references through globals lookup', () => { + it('resolves named loop result bracket paths in function code', async () => { + const loopBlock = createBlock('loop-1', 'Loop 1', 'loop') + const functionBlock = createBlock('function', 'Function', BlockType.FUNCTION, { + language: 'javascript', + }) + const workflow: SerializedWorkflow = { + version: '1', + blocks: [loopBlock, functionBlock], + connections: [], + loops: { 'loop-1': { nodes: ['producer'] } }, + parallels: {}, + } + const state = new ExecutionState() + state.setBlockOutput('loop-1', { + results: [[{ id: 'a' }], [{ id: 'b' }]], + }) + const ctx = { + blockStates: state.getBlockStates(), + blockLogs: [], + environmentVariables: {}, + workflowVariables: {}, + decisions: { router: new Map(), condition: new Map() }, + loopExecutions: new Map(), + executedBlocks: new Set(), + activeExecutionPath: new Set(), + completedLoops: new Set(), + metadata: {}, + } as ExecutionContext + const resolver = new VariableResolver(workflow, {}, state) + + const result = await resolver.resolveInputsForFunctionBlock( + ctx, + 'function', + { code: 'return ' }, + functionBlock + ) + + expect(result.resolvedInputs.code).toBe('return globalThis["__blockRef_0"]') + expect(result.displayInputs.code).toBe('return "b"') + expect(result.contextVariables).toEqual({ __blockRef_0: 'b' }) + }) + + it('rewrites JavaScript file base64 references to lazy runtime reads', async () => { + const { block, ctx, resolver } = createResolver('javascript') + + const result = await resolver.resolveInputsForFunctionBlock( + ctx, + 'function', + { code: 'const base64 = ;\nreturn base64' }, + block + ) + + expect(result.resolvedInputs.code).toBe( + 'const base64 = (await sim.files.readBase64(globalThis["__blockRef_0"]));\nreturn base64' + ) + expect(result.displayInputs.code).toBe('const base64 = ;\nreturn base64') + expect(result.contextVariables.__blockRef_0).toMatchObject({ + id: 'file-1', + name: 'image.png', + }) + expect(result.contextVariables.__blockRef_0).not.toHaveProperty('base64') + }) + + it('wraps lazy JavaScript file base64 reads before member access', async () => { + const { block, ctx, resolver } = createResolver('javascript') + + const result = await resolver.resolveInputsForFunctionBlock( + ctx, + 'function', + { code: 'return .length' }, + block + ) + + expect(result.resolvedInputs.code).toBe( + 'return (await sim.files.readBase64(globalThis["__blockRef_0"])).length' + ) + }) + + it('uses existing inline base64 for keyless files instead of lazy storage reads', async () => { + const { block, ctx, resolver } = createResolver('javascript') + const state = new ExecutionState() + state.setBlockOutput('producer', { + file: { + id: 'file-keyless', + name: 'inline.txt', + key: '', + url: 'https://example.com/inline.txt', + size: 5, + type: 'text/plain', + base64: 'aGVsbG8=', + }, + }) + + const keylessResolver = new VariableResolver( + { + version: '1', + blocks: [createBlock('producer', 'Producer', BlockType.API), block], + connections: [], + loops: {}, + parallels: {}, + }, + {}, + state + ) + + const result = await keylessResolver.resolveInputsForFunctionBlock( + ctx, + 'function', + { code: 'return ' }, + block + ) + + expect(result.resolvedInputs.code).toBe('return globalThis["__blockRef_0"]') + expect(result.contextVariables.__blockRef_0).toBe('aGVsbG8=') + }) + + it('rewrites loop current item base64 references to lazy runtime reads', async () => { + const functionBlock = createBlock('function', 'Function', BlockType.FUNCTION, { + language: 'javascript', + }) + const loopBlock = createBlock('loop-1', 'Loop 1', 'loop') + const workflow: SerializedWorkflow = { + version: '1', + blocks: [loopBlock, functionBlock], + connections: [], + loops: { 'loop-1': { id: 'loop-1', nodes: ['function'], iterations: 1 } }, + parallels: {}, + } + const state = new ExecutionState() + const file = { + id: 'file-loop', + name: 'loop.png', + url: 'https://example.com/loop.png', + key: 'execution/workspace-1/workflow-1/execution-1/loop.png', + context: 'execution', + size: 12 * 1024 * 1024, + type: 'image/png', + base64: 'large-inline-base64', + } + const ctx = { + ...createResolver().ctx, + loopExecutions: new Map([ + [ + 'loop-1', + { + iteration: 0, + currentIterationOutputs: new Map(), + allIterationOutputs: [], + item: file, + items: [file], + }, + ], + ]), + } as ExecutionContext + const resolver = new VariableResolver(workflow, {}, state) + + const result = await resolver.resolveInputsForFunctionBlock( + ctx, + 'function', + { code: 'return .length' }, + functionBlock + ) + + expect(result.resolvedInputs.code).toBe( + 'return (await sim.files.readBase64(globalThis["__blockRef_0"])).length' + ) + expect(result.contextVariables.__blockRef_0).toMatchObject({ id: 'file-loop' }) + expect(result.contextVariables.__blockRef_0).not.toHaveProperty('base64') + }) + + it('rewrites parallel current item base64 references to lazy runtime reads', async () => { + const functionBlock = createBlock('function', 'Function', BlockType.FUNCTION, { + language: 'javascript', + }) + const parallelBlock = createBlock('parallel-1', 'Parallel 1', 'parallel') + const workflow: SerializedWorkflow = { + version: '1', + blocks: [parallelBlock, functionBlock], + connections: [], + loops: {}, + parallels: { + 'parallel-1': { + id: 'parallel-1', + nodes: ['function'], + parallelType: 'collection', + distribution: [], + }, + }, + } + const state = new ExecutionState() + const file = { + id: 'file-parallel', + name: 'parallel.png', + url: 'https://example.com/parallel.png', + key: 'execution/workspace-1/workflow-1/execution-1/parallel.png', + context: 'execution', + size: 12 * 1024 * 1024, + type: 'image/png', + base64: 'large-inline-base64', + } + const ctx = { + ...createResolver().ctx, + parallelExecutions: new Map([ + [ + 'parallel-1', + { + parallelId: 'parallel-1', + totalBranches: 1, + branchOutputs: new Map(), + items: [{ file }], + }, + ], + ]), + parallelBlockMapping: new Map([ + ['function', { originalBlockId: 'function', parallelId: 'parallel-1', iterationIndex: 0 }], + ]), + } as ExecutionContext + const resolver = new VariableResolver(workflow, {}, state) + + const result = await resolver.resolveInputsForFunctionBlock( + ctx, + 'function', + { code: 'return .length' }, + functionBlock + ) + + expect(result.resolvedInputs.code).toBe( + 'return (await sim.files.readBase64(globalThis["__blockRef_0"])).length' + ) + expect(result.contextVariables.__blockRef_0).toMatchObject({ id: 'file-parallel' }) + expect(result.contextVariables.__blockRef_0).not.toHaveProperty('base64') + }) + + it('rewrites JavaScript large value refs to lazy runtime reads', async () => { + const { block, ctx, resolver } = createResolver('javascript') + const state = new ExecutionState() + state.setBlockOutput('producer', { + result: { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'object', + size: 12 * 1024 * 1024, + key: 'execution/workspace-1/workflow-1/execution-1/large-value-lv_ABCDEFGHIJKL.json', + executionId: 'execution-1', + }, + }) + const workflow: SerializedWorkflow = { + version: '1', + blocks: [createBlock('producer', 'Producer', BlockType.API), block], + connections: [], + loops: {}, + parallels: {}, + } + const largeResolver = new VariableResolver(workflow, {}, state) + const largeCtx = { + ...ctx, + blockStates: state.getBlockStates(), + } as ExecutionContext + + const result = await largeResolver.resolveInputsForFunctionBlock( + largeCtx, + 'function', + { code: 'return ' }, + block + ) + + expect(result.resolvedInputs.code).toBe( + 'return (await sim.values.read(globalThis["__blockRef_0"]))' + ) + expect(result.contextVariables.__blockRef_0).toMatchObject({ + __simLargeValueRef: true, + id: 'lv_ABCDEFGHIJKL', + }) + }) + + it('fails whole large value refs for Function runtimes without lazy helpers', async () => { + const { block, ctx } = createResolver('python') + const state = new ExecutionState() + state.setBlockOutput('producer', { + result: { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'object', + size: 12 * 1024 * 1024, + key: 'execution/workspace-1/workflow-1/execution-1/large-value-lv_ABCDEFGHIJKL.json', + executionId: 'execution-1', + }, + }) + const workflow: SerializedWorkflow = { + version: '1', + blocks: [createBlock('producer', 'Producer', BlockType.API), block], + connections: [], + loops: {}, + parallels: {}, + } + const largeResolver = new VariableResolver(workflow, {}, state) + const largeCtx = { + ...ctx, + blockStates: state.getBlockStates(), + } as ExecutionContext + + await expect( + largeResolver.resolveInputsForFunctionBlock( + largeCtx, + 'function', + { code: 'return ' }, + block + ) + ).rejects.toThrow('This execution value is too large to inline') + }) + + it('fails whole large value refs for JavaScript with imports', async () => { + const { block, ctx } = createResolver('javascript') + const state = new ExecutionState() + state.setBlockOutput('producer', { + result: { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'object', + size: 12 * 1024 * 1024, + key: 'execution/workspace-1/workflow-1/execution-1/large-value-lv_ABCDEFGHIJKL.json', + executionId: 'execution-1', + }, + }) + const workflow: SerializedWorkflow = { + version: '1', + blocks: [createBlock('producer', 'Producer', BlockType.API), block], + connections: [], + loops: {}, + parallels: {}, + } + const largeResolver = new VariableResolver(workflow, {}, state) + const largeCtx = { + ...ctx, + blockStates: state.getBlockStates(), + } as ExecutionContext + + await expect( + largeResolver.resolveInputsForFunctionBlock( + largeCtx, + 'function', + { code: "import x from 'x'\nreturn " }, + block + ) + ).rejects.toThrow('This execution value is too large to inline') + }) + + it('keeps JavaScript lazy helpers enabled when import appears in comments or strings', async () => { + const { block, ctx } = createResolver('javascript') + const state = new ExecutionState() + state.setBlockOutput('producer', { + result: { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'object', + size: 12 * 1024 * 1024, + key: 'execution/workspace-1/workflow-1/execution-1/large-value-lv_ABCDEFGHIJKL.json', + executionId: 'execution-1', + }, + }) + const workflow: SerializedWorkflow = { + version: '1', + blocks: [createBlock('producer', 'Producer', BlockType.API), block], + connections: [], + loops: {}, + parallels: {}, + } + const largeResolver = new VariableResolver(workflow, {}, state) + const largeCtx = { + ...ctx, + blockStates: state.getBlockStates(), + } as ExecutionContext + + const result = await largeResolver.resolveInputsForFunctionBlock( + largeCtx, + 'function', + { + code: "/** @import { Foo } from 'foo' */\nconst text = \"import bar from 'bar'\"\nreturn ", + }, + block + ) + + expect(result.resolvedInputs.code).toBe( + '/** @import { Foo } from \'foo\' */\nconst text = "import bar from \'bar\'"\nreturn (await sim.values.read(globalThis["__blockRef_0"]))' + ) + }) + + it('keeps JavaScript lazy helpers enabled for dynamic import expressions', async () => { + const { block, ctx } = createResolver('javascript') + const state = new ExecutionState() + state.setBlockOutput('producer', { + result: { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'object', + size: 12 * 1024 * 1024, + key: 'execution/workspace-1/workflow-1/execution-1/large-value-lv_ABCDEFGHIJKL.json', + executionId: 'execution-1', + }, + }) + const workflow: SerializedWorkflow = { + version: '1', + blocks: [createBlock('producer', 'Producer', BlockType.API), block], + connections: [], + loops: {}, + parallels: {}, + } + const largeResolver = new VariableResolver(workflow, {}, state) + const largeCtx = { + ...ctx, + blockStates: state.getBlockStates(), + } as ExecutionContext + + const result = await largeResolver.resolveInputsForFunctionBlock( + largeCtx, + 'function', + { code: "const mod = import('foo')\nreturn " }, + block + ) + + expect(result.resolvedInputs.code).toBe( + 'const mod = import(\'foo\')\nreturn (await sim.values.read(globalThis["__blockRef_0"]))' + ) + }) + + it('fails nested large value refs for Function runtimes without lazy helpers', async () => { + const { block, ctx } = createResolver('python') + const state = new ExecutionState() + state.setBlockOutput('producer', { + result: { + rows: { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'array', + size: 12 * 1024 * 1024, + key: 'execution/workspace-1/workflow-1/execution-1/large-value-lv_ABCDEFGHIJKL.json', + executionId: 'execution-1', + }, + }, + }) + const workflow: SerializedWorkflow = { + version: '1', + blocks: [createBlock('producer', 'Producer', BlockType.API), block], + connections: [], + loops: {}, + parallels: {}, + } + const largeResolver = new VariableResolver(workflow, {}, state) + const largeCtx = { + ...ctx, + blockStates: state.getBlockStates(), + } as ExecutionContext + + await expect( + largeResolver.resolveInputsForFunctionBlock( + largeCtx, + 'function', + { code: 'return ' }, + block + ) + ).rejects.toThrow('This execution value contains nested large values') + }) + + it('fails nested large value refs for JavaScript instead of leaking ref markers', async () => { + const { block, ctx } = createResolver('javascript') + const state = new ExecutionState() + state.setBlockOutput('producer', { + result: { + rows: { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'array', + size: 12 * 1024 * 1024, + key: 'execution/workspace-1/workflow-1/execution-1/large-value-lv_ABCDEFGHIJKL.json', + executionId: 'execution-1', + }, + }, + }) + const workflow: SerializedWorkflow = { + version: '1', + blocks: [createBlock('producer', 'Producer', BlockType.API), block], + connections: [], + loops: {}, + parallels: {}, + } + const largeResolver = new VariableResolver(workflow, {}, state) + const largeCtx = { + ...ctx, + blockStates: state.getBlockStates(), + } as ExecutionContext + + await expect( + largeResolver.resolveInputsForFunctionBlock( + largeCtx, + 'function', + { code: 'return .rows.length' }, + block + ) + ).rejects.toThrow('This execution value contains nested large values') + }) + + it('resolves Python block references through globals lookup', async () => { const { block, ctx, resolver } = createResolver('python') - const result = resolver.resolveInputsForFunctionBlock( + const result = await resolver.resolveInputsForFunctionBlock( ctx, 'function', { code: 'return ' }, @@ -99,10 +618,10 @@ describe('VariableResolver function block inputs', () => { expect(result.contextVariables).toEqual({ __blockRef_0: 'hello world' }) }) - it('breaks JavaScript string literals around quoted block references', () => { + it('breaks JavaScript string literals around quoted block references', async () => { const { block, ctx, resolver } = createResolver('javascript') - const result = resolver.resolveInputsForFunctionBlock( + const result = await resolver.resolveInputsForFunctionBlock( ctx, 'function', { code: "const rawEmail = '';\nreturn rawEmail" }, @@ -116,10 +635,10 @@ describe('VariableResolver function block inputs', () => { expect(result.contextVariables).toEqual({ __blockRef_0: 'hello world' }) }) - it('uses template interpolation for JavaScript template literal block references', () => { + it('uses template interpolation for JavaScript template literal block references', async () => { const { block, ctx, resolver } = createResolver('javascript') - const result = resolver.resolveInputsForFunctionBlock( + const result = await resolver.resolveInputsForFunctionBlock( ctx, 'function', { code: 'return `value: `' }, @@ -134,10 +653,10 @@ describe('VariableResolver function block inputs', () => { expect(result.contextVariables).toEqual({ __blockRef_0: 'hello world' }) }) - it('keeps JavaScript block references inside template expressions executable', () => { + it('keeps JavaScript block references inside template expressions executable', async () => { const { block, ctx, resolver } = createResolver('javascript') - const result = resolver.resolveInputsForFunctionBlock( + const result = await resolver.resolveInputsForFunctionBlock( ctx, 'function', // biome-ignore lint/suspicious/noTemplateCurlyInString: intentional — asserting template literal is preserved @@ -152,10 +671,10 @@ describe('VariableResolver function block inputs', () => { expect(result.contextVariables).toEqual({ __blockRef_0: 'hello world' }) }) - it('ignores JavaScript comment quotes before later block references', () => { + it('ignores JavaScript comment quotes before later block references', async () => { const { block, ctx, resolver } = createResolver('javascript') - const result = resolver.resolveInputsForFunctionBlock( + const result = await resolver.resolveInputsForFunctionBlock( ctx, 'function', { code: "// don't confuse quote tracking\nreturn " }, @@ -169,10 +688,10 @@ describe('VariableResolver function block inputs', () => { expect(result.contextVariables).toEqual({ __blockRef_0: 'hello world' }) }) - it('breaks Python string literals around quoted block references', () => { + it('breaks Python string literals around quoted block references', async () => { const { block, ctx, resolver } = createResolver('python') - const result = resolver.resolveInputsForFunctionBlock( + const result = await resolver.resolveInputsForFunctionBlock( ctx, 'function', { code: "raw_email = ''\nreturn raw_email" }, @@ -186,10 +705,10 @@ describe('VariableResolver function block inputs', () => { expect(result.contextVariables).toEqual({ __blockRef_0: 'hello world' }) }) - it('breaks Python triple-double-quoted strings around block references', () => { + it('breaks Python triple-double-quoted strings around block references', async () => { const { block, ctx, resolver } = createResolver('python') - const result = resolver.resolveInputsForFunctionBlock( + const result = await resolver.resolveInputsForFunctionBlock( ctx, 'function', { code: 'prompt = """\nSummary: \n"""\nreturn prompt' }, @@ -205,10 +724,10 @@ describe('VariableResolver function block inputs', () => { expect(result.contextVariables).toEqual({ __blockRef_0: 'hello world' }) }) - it('ignores escaped triple-double quotes before later Python block references', () => { + it('ignores escaped triple-double quotes before later Python block references', async () => { const { block, ctx, resolver } = createResolver('python') - const result = resolver.resolveInputsForFunctionBlock( + const result = await resolver.resolveInputsForFunctionBlock( ctx, 'function', { code: 'prompt = """Escaped delimiter: \\"\\"\\"\nSummary: \n"""' }, @@ -224,10 +743,10 @@ describe('VariableResolver function block inputs', () => { expect(result.contextVariables).toEqual({ __blockRef_0: 'hello world' }) }) - it('breaks Python triple-single-quoted strings around block references', () => { + it('breaks Python triple-single-quoted strings around block references', async () => { const { block, ctx, resolver } = createResolver('python') - const result = resolver.resolveInputsForFunctionBlock( + const result = await resolver.resolveInputsForFunctionBlock( ctx, 'function', { code: "prompt = '''\nSummary: \n'''\nreturn prompt" }, @@ -243,10 +762,10 @@ describe('VariableResolver function block inputs', () => { expect(result.contextVariables).toEqual({ __blockRef_0: 'hello world' }) }) - it('ignores Python comment quotes before later block references', () => { + it('ignores Python comment quotes before later block references', async () => { const { block, ctx, resolver } = createResolver('python') - const result = resolver.resolveInputsForFunctionBlock( + const result = await resolver.resolveInputsForFunctionBlock( ctx, 'function', { code: "# don't confuse quote tracking\nreturn " }, @@ -260,10 +779,10 @@ describe('VariableResolver function block inputs', () => { expect(result.contextVariables).toEqual({ __blockRef_0: 'hello world' }) }) - it('uses separate Python context variables for repeated mutable references', () => { + it('uses separate Python context variables for repeated mutable references', async () => { const { block, ctx, resolver } = createResolver('python') - const result = resolver.resolveInputsForFunctionBlock( + const result = await resolver.resolveInputsForFunctionBlock( ctx, 'function', { code: 'a = \nb = \nreturn b' }, @@ -282,10 +801,10 @@ describe('VariableResolver function block inputs', () => { }) }) - it('uses shell-safe expansions for block references', () => { + it('uses shell-safe expansions for block references', async () => { const { block, ctx, resolver } = createResolver('shell') - const result = resolver.resolveInputsForFunctionBlock( + const result = await resolver.resolveInputsForFunctionBlock( ctx, 'function', { code: 'echo suffix && echo ""' }, @@ -302,10 +821,10 @@ describe('VariableResolver function block inputs', () => { }) }) - it('ignores shell comment quotes when formatting later block references', () => { + it('ignores shell comment quotes when formatting later block references', async () => { const { block, ctx, resolver } = createResolver('shell') - const result = resolver.resolveInputsForFunctionBlock( + const result = await resolver.resolveInputsForFunctionBlock( ctx, 'function', { code: "# don't confuse quote tracking\necho " }, diff --git a/apps/sim/executor/variables/resolver.ts b/apps/sim/executor/variables/resolver.ts index c0ab54d23d9..33ac7e181b0 100644 --- a/apps/sim/executor/variables/resolver.ts +++ b/apps/sim/executor/variables/resolver.ts @@ -1,14 +1,22 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { BlockType } from '@/executor/constants' +import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' +import { + containsLargeValueRef, + getLargeValueMaterializationError, + isLargeValueRef, +} from '@/lib/execution/payloads/large-value-ref' +import { isLikelyReferenceSegment } from '@/lib/workflows/sanitization/references' +import { BlockType, parseReferencePath, REFERENCE } from '@/executor/constants' import type { ExecutionState, LoopScope } from '@/executor/execution/state' import type { ExecutionContext } from '@/executor/types' -import { createEnvVarPattern, replaceValidReferences } from '@/executor/utils/reference-validation' +import { createEnvVarPattern, createReferencePattern } from '@/executor/utils/reference-validation' import { BlockResolver } from '@/executor/variables/resolvers/block' import { EnvResolver } from '@/executor/variables/resolvers/env' import { LoopResolver } from '@/executor/variables/resolvers/loop' import { ParallelResolver } from '@/executor/variables/resolvers/parallel' import { + type AsyncPathNavigator, RESOLVED_EMPTY, type ResolutionContext, type Resolver, @@ -23,6 +31,48 @@ export const FUNCTION_BLOCK_DISPLAY_CODE_KEY = '_runtimeDisplayCode' const logger = createLogger('VariableResolver') +function getNestedLargeValueMaterializationError(): Error { + return new Error( + 'This execution value contains nested large values. Reference the nested field directly so it can be lazy-loaded.' + ) +} + +async function replaceValidReferencesAsync( + template: string, + replacer: (match: string, index: number, template: string) => Promise +): Promise { + const pattern = createReferencePattern() + let cursor = 0 + let result = '' + for (const match of template.matchAll(pattern)) { + const fullMatch = match[0] + const index = match.index ?? 0 + result += template.slice(cursor, index) + result += isLikelyReferenceSegment(fullMatch) + ? await replacer(fullMatch, index, template) + : fullMatch + cursor = index + fullMatch.length + } + return result + template.slice(cursor) +} + +async function replaceEnvVarsAsync( + template: string, + replacer: (match: string) => Promise +): Promise { + const pattern = createEnvVarPattern() + let cursor = 0 + let result = '' + for (const match of template.matchAll(pattern)) { + const fullMatch = match[0] + const index = match.index ?? 0 + result += template.slice(cursor, index) + result += await replacer(fullMatch) + cursor = index + fullMatch.length + } + return result + template.slice(cursor) +} + type ShellQuoteContext = 'single' | 'double' | null type CodeStringQuoteContext = ShellQuoteContext | 'triple-single' | 'triple-double' | 'template' type CodeScanMode = @@ -43,12 +93,13 @@ export class VariableResolver { constructor( workflow: SerializedWorkflow, workflowVariables: Record, - private state: ExecutionState + private state: ExecutionState, + options: { navigatePathAsync?: AsyncPathNavigator } = {} ) { - this.blockResolver = new BlockResolver(workflow) + this.blockResolver = new BlockResolver(workflow, options.navigatePathAsync) this.resolvers = [ - new LoopResolver(workflow), - new ParallelResolver(workflow), + new LoopResolver(workflow, options.navigatePathAsync), + new ParallelResolver(workflow, options.navigatePathAsync), new WorkflowResolver(workflowVariables), new EnvResolver(), this.blockResolver, @@ -64,16 +115,16 @@ export class VariableResolver { * should inject contextVariables into the function execution request body so the * isolated VM can access them as global variables. */ - resolveInputsForFunctionBlock( + async resolveInputsForFunctionBlock( ctx: ExecutionContext, currentNodeId: string, params: Record | null | undefined, block: SerializedBlock - ): { + ): Promise<{ resolvedInputs: Record displayInputs: Record contextVariables: Record - } { + }> { const contextVariables: Record = {} const resolved: Record = {} const display: Record = {} @@ -85,7 +136,7 @@ export class VariableResolver { for (const [key, value] of Object.entries(params)) { if (key === 'code') { if (typeof value === 'string') { - const code = this.resolveCodeWithContextVars( + const code = await this.resolveCodeWithContextVars( ctx, currentNodeId, value, @@ -100,7 +151,7 @@ export class VariableResolver { const displayItems: any[] = [] for (const item of value) { if (item && typeof item === 'object' && typeof item.content === 'string') { - const code = this.resolveCodeWithContextVars( + const code = await this.resolveCodeWithContextVars( ctx, currentNodeId, item.content, @@ -124,11 +175,11 @@ export class VariableResolver { resolved[key] = resolvedItems display[key] = displayItems } else { - resolved[key] = this.resolveValue(ctx, currentNodeId, value, undefined, block) + resolved[key] = await this.resolveValue(ctx, currentNodeId, value, undefined, block) display[key] = resolved[key] } } else { - resolved[key] = this.resolveValue(ctx, currentNodeId, value, undefined, block) + resolved[key] = await this.resolveValue(ctx, currentNodeId, value, undefined, block) display[key] = resolved[key] } } @@ -136,12 +187,12 @@ export class VariableResolver { return { resolvedInputs: resolved, displayInputs: display, contextVariables } } - resolveInputs( + async resolveInputs( ctx: ExecutionContext, currentNodeId: string, params: Record, block?: SerializedBlock - ): Record { + ): Promise> { if (!params) { return {} } @@ -152,15 +203,21 @@ export class VariableResolver { try { const parsed = JSON.parse(params.conditions) if (Array.isArray(parsed)) { - resolved.conditions = parsed.map((cond: any) => ({ - ...cond, - value: - typeof cond.value === 'string' - ? this.resolveTemplateWithoutConditionFormatting(ctx, currentNodeId, cond.value) - : cond.value, - })) + resolved.conditions = await Promise.all( + parsed.map(async (cond: any) => ({ + ...cond, + value: + typeof cond.value === 'string' + ? await this.resolveTemplateWithoutConditionFormatting( + ctx, + currentNodeId, + cond.value + ) + : cond.value, + })) + ) } else { - resolved.conditions = this.resolveValue( + resolved.conditions = await this.resolveValue( ctx, currentNodeId, params.conditions, @@ -173,7 +230,7 @@ export class VariableResolver { error: parseError, conditions: params.conditions, }) - resolved.conditions = this.resolveValue( + resolved.conditions = await this.resolveValue( ctx, currentNodeId, params.conditions, @@ -187,17 +244,17 @@ export class VariableResolver { if (isConditionBlock && key === 'conditions') { continue } - resolved[key] = this.resolveValue(ctx, currentNodeId, value, undefined, block) + resolved[key] = await this.resolveValue(ctx, currentNodeId, value, undefined, block) } return resolved } - resolveSingleReference( + async resolveSingleReference( ctx: ExecutionContext, currentNodeId: string, reference: string, loopScope?: LoopScope - ): any { + ): Promise { if (typeof reference === 'string') { const trimmed = reference.trim() if (/^<[^<>]+>$/.test(trimmed)) { @@ -208,7 +265,7 @@ export class VariableResolver { loopScope, } - const result = this.resolveReference(trimmed, resolutionContext) + const result = await this.resolveReference(trimmed, resolutionContext) if (result === RESOLVED_EMPTY) { return null } @@ -219,29 +276,31 @@ export class VariableResolver { return this.resolveValue(ctx, currentNodeId, reference, loopScope) } - private resolveValue( + private async resolveValue( ctx: ExecutionContext, currentNodeId: string, value: any, loopScope?: LoopScope, block?: SerializedBlock - ): any { + ): Promise { if (value === null || value === undefined) { return value } if (Array.isArray(value)) { - return value.map((v) => this.resolveValue(ctx, currentNodeId, v, loopScope, block)) + return Promise.all( + value.map((v) => this.resolveValue(ctx, currentNodeId, v, loopScope, block)) + ) } if (typeof value === 'object') { - return Object.entries(value).reduce( - (acc, [key, val]) => ({ - ...acc, - [key]: this.resolveValue(ctx, currentNodeId, val, loopScope, block), - }), - {} + const entries = await Promise.all( + Object.entries(value).map(async ([key, val]) => [ + key, + await this.resolveValue(ctx, currentNodeId, val, loopScope, block), + ]) ) + return Object.fromEntries(entries) } if (typeof value === 'string') { @@ -256,19 +315,20 @@ export class VariableResolver { * items, workflow variables, env vars) are still inlined as literals so they remain * available without any extra passing mechanism. */ - private resolveCodeWithContextVars( + private async resolveCodeWithContextVars( ctx: ExecutionContext, currentNodeId: string, template: string, loopScope: LoopScope | undefined, block: SerializedBlock, contextVarAccumulator: Record - ): { resolvedCode: string; displayCode: string } { + ): Promise<{ resolvedCode: string; displayCode: string }> { const resolutionContext: ResolutionContext = { executionContext: ctx, executionState: this.state, currentNodeId, loopScope, + allowLargeValueRefs: true, } const language = (block.config?.params as Record | undefined)?.language as @@ -279,14 +339,27 @@ export class VariableResolver { let displayResult = '' let displayCursor = 0 - let result = replaceValidReferences(template, (match, index) => { + let result = await replaceValidReferencesAsync(template, async (match, index) => { if (replacementError) return match displayResult += template.slice(displayCursor, index) displayCursor = index + match.length try { + const lazyBase64 = await this.resolveLazyFileBase64Reference( + match, + resolutionContext, + language, + template, + index, + contextVarAccumulator + ) + if (lazyBase64) { + displayResult += lazyBase64.display + return lazyBase64.replacement + } + if (this.blockResolver.canResolve(match)) { - const resolved = this.resolveReference(match, resolutionContext) + const resolved = await this.resolveReference(match, resolutionContext) if (resolved === undefined) { displayResult += match return match @@ -298,13 +371,29 @@ export class VariableResolver { // with language-specific runtime access to that stored value. const varName = `__blockRef_${Object.keys(contextVarAccumulator).length}` contextVarAccumulator[varName] = effectiveValue - const replacement = this.formatContextVariableReference( - varName, - language, - template, - index, - effectiveValue - ) + let replacement: string + if (isLargeValueRef(effectiveValue)) { + const lazyReplacement = this.formatLazyLargeValueReference( + varName, + language, + template, + index + ) + if (!lazyReplacement) { + throw getLargeValueMaterializationError(effectiveValue) + } + replacement = lazyReplacement + } else if (containsLargeValueRef(effectiveValue)) { + throw getNestedLargeValueMaterializationError() + } else { + replacement = this.formatContextVariableReference( + varName, + language, + template, + index, + effectiveValue + ) + } displayResult += this.formatDisplayValueForCodeContext( effectiveValue, language, @@ -314,7 +403,7 @@ export class VariableResolver { return replacement } - const resolved = this.resolveReference(match, resolutionContext) + const resolved = await this.resolveReference(match, resolutionContext) if (resolved === undefined) { displayResult += match return match @@ -322,6 +411,31 @@ export class VariableResolver { const effectiveValue = resolved === RESOLVED_EMPTY ? null : resolved + if (isLargeValueRef(effectiveValue)) { + const varName = `__blockRef_${Object.keys(contextVarAccumulator).length}` + contextVarAccumulator[varName] = effectiveValue + const lazyReplacement = this.formatLazyLargeValueReference( + varName, + language, + template, + index + ) + if (lazyReplacement) { + displayResult += this.formatDisplayValueForCodeContext( + effectiveValue, + language, + template, + index + ) + return lazyReplacement + } + throw getLargeValueMaterializationError(effectiveValue) + } + + if (containsLargeValueRef(effectiveValue)) { + throw getNestedLargeValueMaterializationError() + } + // Non-block reference (loop, parallel, workflow, env): embed as literal const replacement = this.blockResolver.formatValueForBlock( effectiveValue, @@ -342,18 +456,241 @@ export class VariableResolver { throw replacementError } - result = result.replace(createEnvVarPattern(), (match) => { - const resolved = this.resolveReference(match, resolutionContext) + result = await replaceEnvVarsAsync(result, async (match) => { + const resolved = await this.resolveReference(match, resolutionContext) return typeof resolved === 'string' ? resolved : match }) - displayResult = displayResult.replace(createEnvVarPattern(), (match) => { - const resolved = this.resolveReference(match, resolutionContext) + displayResult = await replaceEnvVarsAsync(displayResult, async (match) => { + const resolved = await this.resolveReference(match, resolutionContext) return typeof resolved === 'string' ? resolved : match }) return { resolvedCode: result, displayCode: displayResult } } + private async resolveLazyFileBase64Reference( + reference: string, + context: ResolutionContext, + language: string | undefined, + template: string, + matchIndex: number, + contextVarAccumulator: Record + ): Promise<{ replacement: string; display: string } | null> { + if (!this.canUseJavaScriptRuntimeHelpers(language, template)) { + return null + } + + const parts = parseReferencePath(reference) + if (parts.length < 3 || parts.at(-1) !== 'base64') { + return null + } + + const fileReference = `${REFERENCE.START}${parts.slice(0, -1).join(REFERENCE.PATH_DELIMITER)}${REFERENCE.END}` + const file = await this.resolveReference(fileReference, context) + if (!isUserFileWithMetadata(file)) { + return null + } + if (!file.key) { + return null + } + + const varName = `__blockRef_${Object.keys(contextVarAccumulator).length}` + const { base64: _base64, ...fileMetadata } = file + contextVarAccumulator[varName] = fileMetadata + const fileExpression = `globalThis[${JSON.stringify(varName)}]` + const lazyExpression = `(await sim.files.readBase64(${fileExpression}))` + + return { + replacement: this.formatJavaScriptAsyncExpression(lazyExpression, template, matchIndex), + display: reference, + } + } + + private formatLazyLargeValueReference( + varName: string, + language: string | undefined, + template: string, + matchIndex: number + ): string | null { + if (!this.canUseJavaScriptRuntimeHelpers(language, template)) { + return null + } + + const expression = `(await sim.values.read(globalThis[${JSON.stringify(varName)}]))` + return this.formatJavaScriptAsyncExpression(expression, template, matchIndex, { + stringifyInStringContext: true, + }) + } + + private formatJavaScriptAsyncExpression( + expression: string, + template: string, + matchIndex: number, + options: { stringifyInStringContext?: boolean } = {} + ): string { + const quoteContext = this.getCodeStringQuoteContext(template, matchIndex, 'javascript') + const stringExpression = options.stringifyInStringContext + ? `JSON.stringify(${expression})` + : expression + + if (quoteContext === 'template') { + return `\${${stringExpression}}` + } + if (quoteContext === 'single' || quoteContext === 'double') { + const quote = this.getCodeStringQuoteToken(quoteContext) + return `${quote} + ${stringExpression} + ${quote}` + } + return expression + } + + private canUseJavaScriptRuntimeHelpers(language: string | undefined, template: string): boolean { + if (language !== 'javascript') { + return false + } + return !this.hasJavaScriptModuleDependencySyntax(template) + } + + private hasJavaScriptModuleDependencySyntax(template: string): boolean { + const modes: CodeScanMode[] = [{ type: 'normal' }] + + for (let i = 0; i < template.length; i++) { + const char = template[i] + const next = template[i + 1] + const mode = modes[modes.length - 1] + + if (mode.type === 'line-comment') { + if (char === '\n') modes.pop() + continue + } + + if (mode.type === 'block-comment') { + if (char === '*' && next === '/') { + modes.pop() + i++ + } + continue + } + + if (mode.type === 'single' || mode.type === 'double') { + const quote = mode.type === 'single' ? "'" : '"' + if (char === '\\') { + i++ + continue + } + if (char === quote || char === '\n') modes.pop() + continue + } + + if (mode.type === 'template') { + if (char === '\\') { + i++ + continue + } + if (char === '`') { + modes.pop() + continue + } + if (char === '$' && next === '{') { + modes.push({ type: 'template-expression', depth: 1 }) + i++ + } + continue + } + + const isCodeMode = mode.type === 'normal' || mode.type === 'template-expression' + if (!isCodeMode) continue + + if (char === '/' && next === '/') { + modes.push({ type: 'line-comment' }) + i++ + continue + } + if (char === '/' && next === '*') { + modes.push({ type: 'block-comment' }) + i++ + continue + } + if (char === "'") { + modes.push({ type: 'single' }) + continue + } + if (char === '"') { + modes.push({ type: 'double' }) + continue + } + if (char === '`') { + modes.push({ type: 'template' }) + continue + } + + if (mode.type === 'template-expression') { + if (char === '{') { + mode.depth += 1 + continue + } + if (char === '}') { + mode.depth -= 1 + if (mode.depth === 0) modes.pop() + continue + } + } + + if (this.startsWithStaticImport(template, i) || this.startsWithRequireCall(template, i)) { + return true + } + } + + return false + } + + private startsWithStaticImport(template: string, index: number): boolean { + if (!this.matchesKeywordAt(template, index, 'import')) { + return false + } + const nextIndex = this.skipWhitespace(template, index + 'import'.length) + if (nextIndex === index + 'import'.length) { + return false + } + return template[nextIndex] !== '(' + } + + private startsWithRequireCall(template: string, index: number): boolean { + if (!this.matchesKeywordAt(template, index, 'require')) { + return false + } + const openParenIndex = this.skipWhitespace(template, index + 'require'.length) + if (template[openParenIndex] !== '(') { + return false + } + const argumentIndex = this.skipWhitespace(template, openParenIndex + 1) + return ( + template[argumentIndex] === "'" || + template[argumentIndex] === '"' || + template[argumentIndex] === '`' + ) + } + + private matchesKeywordAt(template: string, index: number, keyword: string): boolean { + if (!template.startsWith(keyword, index)) { + return false + } + const before = index > 0 ? template[index - 1] : '' + const after = template[index + keyword.length] ?? '' + return !this.isJavaScriptIdentifierChar(before) && !this.isJavaScriptIdentifierChar(after) + } + + private skipWhitespace(template: string, index: number): number { + let cursor = index + while (cursor < template.length && /\s/.test(template[cursor])) { + cursor++ + } + return cursor + } + + private isJavaScriptIdentifierChar(char: string): boolean { + return /[A-Za-z0-9_$]/.test(char) + } + private formatContextVariableReference( varName: string, language: string | undefined, @@ -669,13 +1006,13 @@ export class VariableResolver { return previous === undefined || /\s|[;&|()<>]/.test(previous) } - private resolveTemplate( + private async resolveTemplate( ctx: ExecutionContext, currentNodeId: string, template: string, loopScope?: LoopScope, block?: SerializedBlock - ): string { + ): Promise { const resolutionContext: ResolutionContext = { executionContext: ctx, executionState: this.state, @@ -693,11 +1030,11 @@ export class VariableResolver { | undefined) : undefined - let result = replaceValidReferences(template, (match) => { + let result = await replaceValidReferencesAsync(template, async (match) => { if (replacementError) return match try { - const resolved = this.resolveReference(match, resolutionContext) + const resolved = await this.resolveReference(match, resolutionContext) if (resolved === undefined) { return match } @@ -720,19 +1057,19 @@ export class VariableResolver { throw replacementError } - result = result.replace(createEnvVarPattern(), (match) => { - const resolved = this.resolveReference(match, resolutionContext) + result = await replaceEnvVarsAsync(result, async (match) => { + const resolved = await this.resolveReference(match, resolutionContext) return typeof resolved === 'string' ? resolved : match }) return result } - private resolveTemplateWithoutConditionFormatting( + private async resolveTemplateWithoutConditionFormatting( ctx: ExecutionContext, currentNodeId: string, template: string, loopScope?: LoopScope - ): string { + ): Promise { const resolutionContext: ResolutionContext = { executionContext: ctx, executionState: this.state, @@ -742,11 +1079,11 @@ export class VariableResolver { let replacementError: Error | null = null - let result = replaceValidReferences(template, (match) => { + let result = await replaceValidReferencesAsync(template, async (match) => { if (replacementError) return match try { - const resolved = this.resolveReference(match, resolutionContext) + const resolved = await this.resolveReference(match, resolutionContext) if (resolved === undefined) { return match } @@ -779,17 +1116,19 @@ export class VariableResolver { throw replacementError } - result = result.replace(createEnvVarPattern(), (match) => { - const resolved = this.resolveReference(match, resolutionContext) + result = await replaceEnvVarsAsync(result, async (match) => { + const resolved = await this.resolveReference(match, resolutionContext) return typeof resolved === 'string' ? resolved : match }) return result } - private resolveReference(reference: string, context: ResolutionContext): any { + private async resolveReference(reference: string, context: ResolutionContext): Promise { for (const resolver of this.resolvers) { if (resolver.canResolve(reference)) { - const result = resolver.resolve(reference, context) + const result = resolver.resolveAsync + ? await resolver.resolveAsync(reference, context) + : resolver.resolve(reference, context) return result } } diff --git a/apps/sim/executor/variables/resolvers/block.test.ts b/apps/sim/executor/variables/resolvers/block.test.ts index 5b9ed37fc3b..a739dc28734 100644 --- a/apps/sim/executor/variables/resolvers/block.test.ts +++ b/apps/sim/executor/variables/resolvers/block.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from 'vitest' +import { compactExecutionPayload } from '@/lib/execution/payloads/serializer' import { ExecutionState } from '@/executor/execution/state' import { BlockResolver } from './block' import { RESOLVED_EMPTY, type ResolutionContext } from './reference' @@ -174,6 +175,9 @@ function createTestContext( return { executionContext: { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', blockStates: contextBlockStates ?? new Map(), }, executionState: state, @@ -247,6 +251,45 @@ describe('BlockResolver', () => { expect(resolver.resolve('', ctx)).toBe('alice@test.com') }) + it('should resolve nested scalar paths inside compacted block references', async () => { + const workflow = createTestWorkflow([{ id: 'source' }]) + const resolver = new BlockResolver(workflow) + const compacted = await compactExecutionPayload( + { + user: { profile: { name: 'Alice' } }, + items: Array.from({ length: 100 }, (_, index) => ({ id: index })), + }, + { + thresholdBytes: 64, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + } + ) + const ctx = createTestContext('current', { source: compacted }) + + expect(resolver.resolve('', ctx)).toBe('Alice') + expect(resolver.resolve('', ctx)).toBe(1) + expect(() => resolver.resolve('', ctx)).toThrow('too large to inline') + }) + + it('should reject full container references that contain compacted children', async () => { + const workflow = createTestWorkflow([{ id: 'source' }]) + const resolver = new BlockResolver(workflow) + const compacted = await compactExecutionPayload( + { + metadata: { id: 'event-1' }, + attachment: { body: 'x'.repeat(2048) }, + }, + { thresholdBytes: 256, preserveRoot: true } + ) + const ctx = createTestContext('current', { source: compacted }) + + expect(resolver.resolve('', ctx)).toBe('event-1') + expect(() => resolver.resolve('', ctx)).toThrow('too large to inline') + expect(() => resolver.resolve('', ctx)).toThrow('too large to inline') + }) + it.concurrent('should resolve array index in path', () => { const workflow = createTestWorkflow([{ id: 'source' }]) const resolver = new BlockResolver(workflow) diff --git a/apps/sim/executor/variables/resolvers/block.ts b/apps/sim/executor/variables/resolvers/block.ts index e1a5be03f7a..7a03093e6a1 100644 --- a/apps/sim/executor/variables/resolvers/block.ts +++ b/apps/sim/executor/variables/resolvers/block.ts @@ -1,3 +1,4 @@ +import { assertNoLargeValueRefs } from '@/lib/execution/payloads/large-value-ref' import { isReference, normalizeName, @@ -9,9 +10,11 @@ import { InvalidFieldError, type OutputSchema, resolveBlockReference, + resolveBlockReferenceAsync, } from '@/executor/utils/block-reference' import { formatLiteralForCode } from '@/executor/utils/code-formatting' import { + type AsyncPathNavigator, navigatePath, RESOLVED_EMPTY, type ResolutionContext, @@ -23,7 +26,10 @@ export class BlockResolver implements Resolver { private nameToBlockId: Map private blockById: Map - constructor(private workflow: SerializedWorkflow) { + constructor( + private workflow: SerializedWorkflow, + private navigatePathAsync?: AsyncPathNavigator + ) { this.nameToBlockId = new Map() this.blockById = new Map() for (const block of workflow.blocks) { @@ -75,17 +81,97 @@ export class BlockResolver implements Resolver { } try { - const result = resolveBlockReference(blockName, pathParts, { + const result = resolveBlockReference( + blockName, + pathParts, + { + blockNameMapping: Object.fromEntries(this.nameToBlockId), + blockData, + blockOutputSchemas, + }, + { + allowLargeValueRefs: context.allowLargeValueRefs, + executionContext: context.executionContext, + } + )! + + if (result.value !== undefined) { + if (!context.allowLargeValueRefs) { + assertNoLargeValueRefs(result.value) + } + return result.value + } + + const backwardsCompat = this.handleBackwardsCompatSync(block, output, pathParts) + if (backwardsCompat !== undefined) { + return backwardsCompat + } + + return RESOLVED_EMPTY + } catch (error) { + if (error instanceof InvalidFieldError) { + const fallback = this.handleBackwardsCompatSync(block, output, pathParts) + if (fallback !== undefined) { + return fallback + } + } + throw error + } + } + + async resolveAsync(reference: string, context: ResolutionContext): Promise { + if (!this.navigatePathAsync) { + return this.resolve(reference, context) + } + const parts = parseReferencePath(reference) + if (parts.length === 0) { + return undefined + } + const [blockName, ...pathParts] = parts + + const blockId = this.findBlockIdByName(blockName) + if (!blockId) { + return undefined + } + + const block = this.blockById.get(blockId)! + const output = this.getBlockOutput(blockId, context) + + const blockData: Record = {} + const blockOutputSchemas: Record = {} + + if (output !== undefined) { + blockData[blockId] = output + } + + const outputSchema = getBlockSchema(block) + + if (outputSchema && Object.keys(outputSchema).length > 0) { + blockOutputSchemas[blockId] = outputSchema + } + + try { + const blockReferenceContext = { blockNameMapping: Object.fromEntries(this.nameToBlockId), blockData, blockOutputSchemas, - })! + } + const result = (await resolveBlockReferenceAsync( + blockName, + pathParts, + blockReferenceContext, + context, + this.navigatePathAsync + ))! if (result.value !== undefined) { + if (!context.allowLargeValueRefs) { + assertNoLargeValueRefs(result.value) + } return result.value } - const backwardsCompat = this.handleBackwardsCompat(block, output, pathParts) + const backwardsCompat = await this.handleBackwardsCompat(block, output, pathParts, context) if (backwardsCompat !== undefined) { return backwardsCompat } @@ -93,7 +179,7 @@ export class BlockResolver implements Resolver { return RESOLVED_EMPTY } catch (error) { if (error instanceof InvalidFieldError) { - const fallback = this.handleBackwardsCompat(block, output, pathParts) + const fallback = await this.handleBackwardsCompat(block, output, pathParts, context) if (fallback !== undefined) { return fallback } @@ -102,7 +188,7 @@ export class BlockResolver implements Resolver { } } - private handleBackwardsCompat( + private handleBackwardsCompatSync( block: SerializedBlock, output: unknown, pathParts: string[] @@ -126,6 +212,56 @@ export class BlockResolver implements Resolver { } } + const outputRecord = output as Record | undefined + if ( + (block.metadata?.id === 'workflow' || block.metadata?.id === 'workflow_input') && + pathParts[0] === 'result' && + pathParts[1] === 'response' && + outputRecord?.result !== undefined && + typeof outputRecord.result === 'object' && + outputRecord.result !== null && + (outputRecord.result as Record)?.response === undefined + ) { + const adjustedPathParts = ['result', ...pathParts.slice(2)] + const fallbackResult = navigatePath(output, adjustedPathParts) + if (fallbackResult !== undefined) { + return fallbackResult + } + } + + return undefined + } + + private async handleBackwardsCompat( + block: SerializedBlock, + output: unknown, + pathParts: string[], + context: ResolutionContext + ): Promise { + const navigatePathAsync = this.navigatePathAsync + if (!navigatePathAsync) { + return this.handleBackwardsCompatSync(block, output, pathParts) + } + + if (output === undefined || pathParts.length === 0) { + return undefined + } + + if ( + block.metadata?.id === 'response' && + pathParts[0] === 'response' && + (output as Record)?.response === undefined + ) { + const adjustedPathParts = pathParts.slice(1) + if (adjustedPathParts.length === 0) { + return output + } + const fallbackResult = await navigatePathAsync(output, adjustedPathParts, context) + if (fallbackResult !== undefined) { + return fallbackResult + } + } + const isWorkflowBlock = block.metadata?.id === 'workflow' || block.metadata?.id === 'workflow_input' const outputRecord = output as Record | undefined> @@ -136,7 +272,7 @@ export class BlockResolver implements Resolver { outputRecord?.result?.response === undefined ) { const adjustedPathParts = ['result', ...pathParts.slice(2)] - const fallbackResult = navigatePath(output, adjustedPathParts) + const fallbackResult = await navigatePathAsync(output, adjustedPathParts, context) if (fallbackResult !== undefined) { return fallbackResult } diff --git a/apps/sim/executor/variables/resolvers/loop.test.ts b/apps/sim/executor/variables/resolvers/loop.test.ts index 3d3b643b516..48576ffc67a 100644 --- a/apps/sim/executor/variables/resolvers/loop.test.ts +++ b/apps/sim/executor/variables/resolvers/loop.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest' +import { compactExecutionPayload } from '@/lib/execution/payloads/serializer' import type { LoopScope } from '@/executor/execution/state' import { InvalidFieldError } from '@/executor/utils/block-reference' import { LoopResolver } from './loop' @@ -61,6 +62,9 @@ function createTestContext( ): ResolutionContext { return { executionContext: { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', loopExecutions: loopExecutions ?? new Map(), }, executionState: { @@ -232,6 +236,9 @@ describe('LoopResolver', () => { const ctx = createTestContext('block-1', loopScope) expect(() => resolver.resolve('', ctx)).toThrow(InvalidFieldError) + expect(() => resolver.resolve('', ctx)).toThrow( + 'Available fields: index' + ) }) it.concurrent('should handle iteration index 0 correctly', () => { @@ -361,7 +368,7 @@ describe('LoopResolver', () => { expect(resolver.resolve('', ctx)).toBe(4) }) - it.concurrent('should return undefined for index when block is outside the loop', () => { + it.concurrent('should throw for contextual fields when block is outside the loop', () => { const workflow = createTestWorkflow({ 'loop-1': { nodes: ['block-1'] } }, [ { id: 'loop-1', name: 'Loop 1' }, ]) @@ -370,7 +377,8 @@ describe('LoopResolver', () => { const loopExecutions = new Map([['loop-1', loopScope]]) const ctx = createTestContext('block-outside', undefined, loopExecutions) - expect(resolver.resolve('', ctx)).toBeUndefined() + expect(() => resolver.resolve('', ctx)).toThrow(InvalidFieldError) + expect(() => resolver.resolve('', ctx)).toThrow('Available fields: results') }) it.concurrent('should resolve result from anywhere after loop completes', () => { @@ -399,6 +407,30 @@ describe('LoopResolver', () => { expect(resolver.resolve('', ctx)).toEqual([{ response: 'a' }]) expect(resolver.resolve('', ctx)).toBe('b') + expect(resolver.resolve('', ctx)).toBe('b') + }) + + it('should resolve nested paths inside compacted result references', async () => { + const workflow = createTestWorkflow({ 'loop-1': { nodes: ['block-1'] } }, [ + { id: 'loop-1', name: 'Loop 1' }, + ]) + const resolver = new LoopResolver(workflow) + const compacted = await compactExecutionPayload( + { results: [[{ response: 'a' }], [{ response: 'b', payload: 'x'.repeat(2048) }]] }, + { + thresholdBytes: 256, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + } + ) + const ctx = createTestContext('block-outside', undefined, new Map(), { + 'loop-1': compacted, + }) + + expect(resolver.resolve('', ctx)).toBe('b') + expect(resolver.resolve('', ctx)).toBe('b') + expect(() => resolver.resolve('', ctx)).toThrow('too large to inline') }) it.concurrent('should resolve forEach properties via named reference', () => { @@ -427,6 +459,20 @@ describe('LoopResolver', () => { const ctx = createTestContext('block-1', undefined, loopExecutions) expect(() => resolver.resolve('', ctx)).toThrow(InvalidFieldError) + expect(() => resolver.resolve('', ctx)).toThrow('Available fields: index') + }) + + it.concurrent('should list only results for unknown fields outside a named loop', () => { + const workflow = createTestWorkflow({ 'loop-1': { nodes: ['block-1'] } }, [ + { id: 'loop-1', name: 'Loop 1' }, + ]) + const resolver = new LoopResolver(workflow) + const loopScope = createLoopScope({ iteration: 0 }) + const loopExecutions = new Map([['loop-1', loopScope]]) + const ctx = createTestContext('block-outside', undefined, loopExecutions) + + expect(() => resolver.resolve('', ctx)).toThrow(InvalidFieldError) + expect(() => resolver.resolve('', ctx)).toThrow('Available fields: results') }) it.concurrent('should not resolve named ref when no matching block exists', () => { diff --git a/apps/sim/executor/variables/resolvers/loop.ts b/apps/sim/executor/variables/resolvers/loop.ts index 8df57668825..3b0a3e1b611 100644 --- a/apps/sim/executor/variables/resolvers/loop.ts +++ b/apps/sim/executor/variables/resolvers/loop.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { assertNoLargeValueRefs } from '@/lib/execution/payloads/large-value-ref' import { isReference, normalizeName, parseReferencePath, REFERENCE } from '@/executor/constants' import { InvalidFieldError } from '@/executor/utils/block-reference' import { @@ -7,18 +8,26 @@ import { stripOuterBranchSuffix, } from '@/executor/utils/subflow-utils' import { + type AsyncPathNavigator, navigatePath, type ResolutionContext, type Resolver, + splitLeadingBracketPath, } from '@/executor/variables/resolvers/reference' import type { SerializedWorkflow } from '@/serializer/types' const logger = createLogger('LoopResolver') +const LOOP_OUTPUT_FIELDS = ['results'] as const +const LOOP_CONTEXT_FIELDS = ['index'] as const +const FOR_EACH_LOOP_CONTEXT_FIELDS = ['index', 'currentItem', 'items'] as const export class LoopResolver implements Resolver { private loopNameToId: Map - constructor(private workflow: SerializedWorkflow) { + constructor( + private workflow: SerializedWorkflow, + private navigatePathAsync?: AsyncPathNavigator + ) { this.loopNameToId = new Map() for (const block of workflow.blocks) { if (workflow.loops[block.id] && block.metadata?.name) { @@ -43,6 +52,27 @@ export class LoopResolver implements Resolver { } resolve(reference: string, context: ResolutionContext): any { + return this.resolveInternal(reference, context, false) + } + + async resolveAsync(reference: string, context: ResolutionContext): Promise { + if (!this.navigatePathAsync) { + return this.resolve(reference, context) + } + return this.resolveInternal(reference, context, true) + } + + private async resolveInternal( + reference: string, + context: ResolutionContext, + useAsyncPath: true + ): Promise + private resolveInternal(reference: string, context: ResolutionContext, useAsyncPath: false): any + private resolveInternal( + reference: string, + context: ResolutionContext, + useAsyncPath: boolean + ): any | Promise { const parts = parseReferencePath(reference) if (parts.length === 0) { logger.warn('Invalid loop reference', { reference }) @@ -76,34 +106,32 @@ export class LoopResolver implements Resolver { } if (rest.length > 0) { - const property = rest[0] + const { property, pathParts: bracketPathParts } = splitLeadingBracketPath(rest[0]) if (LoopResolver.OUTPUT_PROPERTIES.has(property)) { if (!targetLoopId) { return undefined } - return this.resolveOutput(targetLoopId, rest.slice(1), context) + return useAsyncPath + ? this.resolveOutputAsync(targetLoopId, [...bracketPathParts, ...rest.slice(1)], context) + : this.resolveOutput(targetLoopId, [...bracketPathParts, ...rest.slice(1)], context) } + const isContextual = + isGenericRef || + (targetLoopId !== undefined && + this.isBlockInLoopOrDescendant(context.currentNodeId, targetLoopId)) + if (!LoopResolver.KNOWN_PROPERTIES.has(property)) { - const isForEach = targetLoopId - ? this.isForEachLoop(targetLoopId) - : context.loopScope?.items !== undefined - const availableFields = isForEach - ? ['index', 'currentItem', 'items', 'result'] - : ['index', 'result'] - throw new InvalidFieldError(firstPart, property, availableFields) + throw new InvalidFieldError( + firstPart, + rest[0], + this.getAvailableFields(targetLoopId, context) + ) } - if (!isGenericRef && targetLoopId) { - if (!this.isBlockInLoopOrDescendant(context.currentNodeId, targetLoopId)) { - logger.warn('Block is not inside the referenced loop', { - reference, - blockId: context.currentNodeId, - loopId: targetLoopId, - }) - return undefined - } + if (!isContextual) { + throw new InvalidFieldError(firstPart, rest[0], [...LOOP_OUTPUT_FIELDS]) } } @@ -130,7 +158,9 @@ export class LoopResolver implements Resolver { return obj } - const [property, ...pathParts] = rest + const [rawProperty, ...remainingPathParts] = rest + const { property, pathParts: bracketPathParts } = splitLeadingBracketPath(rawProperty) + const pathParts = [...bracketPathParts, ...remainingPathParts] let value: any switch (property) { @@ -148,7 +178,9 @@ export class LoopResolver implements Resolver { } if (pathParts.length > 0) { - return navigatePath(value, pathParts) + return useAsyncPath && this.navigatePathAsync + ? this.navigatePathAsync(value, pathParts, context) + : navigatePath(value, pathParts, { executionContext: context.executionContext }) } return value @@ -161,7 +193,31 @@ export class LoopResolver implements Resolver { } const value = (output as Record).results if (pathParts.length > 0) { - return navigatePath(value, pathParts) + return navigatePath(value, pathParts, { executionContext: context.executionContext }) + } + if (!context.allowLargeValueRefs) { + assertNoLargeValueRefs(value) + } + return value + } + + private async resolveOutputAsync( + loopId: string, + pathParts: string[], + context: ResolutionContext + ): Promise { + const output = context.executionState.getBlockOutput(loopId) + if (!output || typeof output !== 'object') { + return undefined + } + const value = (output as Record).results + if (pathParts.length > 0) { + return this.navigatePathAsync + ? this.navigatePathAsync(value, pathParts, context) + : navigatePath(value, pathParts, { executionContext: context.executionContext }) + } + if (!context.allowLargeValueRefs) { + assertNoLargeValueRefs(value) } return value } @@ -234,4 +290,22 @@ export class LoopResolver implements Resolver { const loopConfig = this.workflow.loops?.[originalId] return loopConfig?.loopType === 'forEach' } + + private getAvailableFields( + targetLoopId: string | undefined, + context: ResolutionContext + ): string[] { + const isContextual = + targetLoopId === undefined || + this.isBlockInLoopOrDescendant(context.currentNodeId, targetLoopId) + + if (!isContextual) { + return [...LOOP_OUTPUT_FIELDS] + } + + const isForEach = targetLoopId + ? this.isForEachLoop(targetLoopId) + : context.loopScope?.items !== undefined + return isForEach ? [...FOR_EACH_LOOP_CONTEXT_FIELDS] : [...LOOP_CONTEXT_FIELDS] + } } diff --git a/apps/sim/executor/variables/resolvers/parallel.test.ts b/apps/sim/executor/variables/resolvers/parallel.test.ts index cec6294f391..3d4764acd4a 100644 --- a/apps/sim/executor/variables/resolvers/parallel.test.ts +++ b/apps/sim/executor/variables/resolvers/parallel.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest' +import { compactExecutionPayload } from '@/lib/execution/payloads/serializer' import { InvalidFieldError } from '@/executor/utils/block-reference' import { ParallelResolver } from './parallel' import type { ResolutionContext } from './reference' @@ -76,11 +77,16 @@ function createParallelScope(items: any[]) { function createTestContext( currentNodeId: string, parallelExecutions?: Map, - blockOutputs?: Record + blockOutputs?: Record, + parallelBlockMapping?: Map ): ResolutionContext { return { executionContext: { + workflowId: 'workflow-1', + workspaceId: 'workspace-1', + executionId: 'execution-1', parallelExecutions: parallelExecutions ?? new Map(), + parallelBlockMapping, }, executionState: { getBlockOutput: (id: string) => blockOutputs?.[id], @@ -158,6 +164,34 @@ describe('ParallelResolver', () => { expect(resolver.resolve('', createTestContext('block-1₍2₎'))).toBe(2) }) + it.concurrent('uses runtime branch mapping for batched local branch node IDs', () => { + const workflow = createTestWorkflow({ + 'parallel-1': { nodes: ['block-1'], distribution: ['a', 'b', 'c', 'd'] }, + }) + const resolver = new ParallelResolver(workflow) + const parallelScope = createParallelScope(['a', 'b', 'c', 'd']) + const parallelExecutions = new Map([['parallel-1', parallelScope]]) + const parallelBlockMapping = new Map([ + [ + 'block-1₍0₎', + { + originalBlockId: 'block-1', + parallelId: 'parallel-1', + iterationIndex: 2, + }, + ], + ]) + const ctx = createTestContext( + 'block-1₍0₎', + parallelExecutions, + undefined, + parallelBlockMapping + ) + + expect(resolver.resolve('', ctx)).toBe(2) + expect(resolver.resolve('', ctx)).toBe('c') + }) + it.concurrent('should return undefined when branch index cannot be extracted', () => { const workflow = createTestWorkflow({ 'parallel-1': { nodes: ['block-1'], distribution: ['a', 'b'] }, @@ -313,6 +347,9 @@ describe('ParallelResolver', () => { const ctx = createTestContext('block-1₍0₎') expect(() => resolver.resolve('', ctx)).toThrow(InvalidFieldError) + expect(() => resolver.resolve('', ctx)).toThrow( + 'Available fields: index' + ) }) it.concurrent('should return undefined when block is not in any parallel', () => { @@ -428,6 +465,31 @@ describe('ParallelResolver', () => { expect(resolver.resolve('', ctx)).toEqual([{ response: 'a' }]) expect(resolver.resolve('', ctx)).toBe('b') + expect(resolver.resolve('', ctx)).toBe('b') + }) + + it('should resolve nested paths inside compacted result references', async () => { + const workflow = createTestWorkflow( + { 'parallel-1': { nodes: ['block-1'], distribution: ['a', 'b'] } }, + [{ id: 'parallel-1', name: 'Parallel 1' }] + ) + const resolver = new ParallelResolver(workflow) + const compacted = await compactExecutionPayload( + { results: [[{ response: 'a' }], [{ response: 'b', payload: 'x'.repeat(2048) }]] }, + { + thresholdBytes: 256, + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + } + ) + const ctx = createTestContext('block-outside', new Map(), { + 'parallel-1': compacted, + }) + + expect(resolver.resolve('', ctx)).toBe('b') + expect(resolver.resolve('', ctx)).toBe('b') + expect(() => resolver.resolve('', ctx)).toThrow('too large to inline') }) it.concurrent('should resolve result with empty currentNodeId', () => { @@ -489,6 +551,29 @@ describe('ParallelResolver', () => { const ctx = createTestContext('block-1₍0₎') expect(() => resolver.resolve('', ctx)).toThrow(InvalidFieldError) + expect(() => resolver.resolve('', ctx)).toThrow( + 'Available fields: index, currentItem, items' + ) + }) + + it.concurrent('should list only results for contextual fields outside a named parallel', () => { + const workflow = createTestWorkflow( + { + 'parallel-1': { + nodes: ['block-1'], + distribution: ['a'], + parallelType: 'collection', + }, + }, + [{ id: 'parallel-1', name: 'Parallel 1' }] + ) + const resolver = new ParallelResolver(workflow) + const ctx = createTestContext('block-outside', new Map()) + + expect(() => resolver.resolve('', ctx)).toThrow(InvalidFieldError) + expect(() => resolver.resolve('', ctx)).toThrow('Available fields: results') + expect(() => resolver.resolve('', ctx)).toThrow(InvalidFieldError) + expect(() => resolver.resolve('', ctx)).toThrow('Available fields: results') }) it.concurrent('should not resolve named ref when no matching block exists', () => { diff --git a/apps/sim/executor/variables/resolvers/parallel.ts b/apps/sim/executor/variables/resolvers/parallel.ts index 7afeedece97..538fc69780d 100644 --- a/apps/sim/executor/variables/resolvers/parallel.ts +++ b/apps/sim/executor/variables/resolvers/parallel.ts @@ -1,25 +1,35 @@ import { createLogger } from '@sim/logger' +import { assertNoLargeValueRefs } from '@/lib/execution/payloads/large-value-ref' import { isReference, normalizeName, parseReferencePath, REFERENCE } from '@/executor/constants' import { InvalidFieldError } from '@/executor/utils/block-reference' import { extractBranchIndex, + extractOuterBranchIndex, findEffectiveContainerId, stripCloneSuffixes, stripOuterBranchSuffix, } from '@/executor/utils/subflow-utils' import { + type AsyncPathNavigator, navigatePath, type ResolutionContext, type Resolver, + splitLeadingBracketPath, } from '@/executor/variables/resolvers/reference' import type { SerializedParallel, SerializedWorkflow } from '@/serializer/types' const logger = createLogger('ParallelResolver') +const PARALLEL_OUTPUT_FIELDS = ['results'] as const +const PARALLEL_CONTEXT_FIELDS = ['index'] as const +const COLLECTION_PARALLEL_CONTEXT_FIELDS = ['index', 'currentItem', 'items'] as const export class ParallelResolver implements Resolver { private parallelNameToId: Map - constructor(private workflow: SerializedWorkflow) { + constructor( + private workflow: SerializedWorkflow, + private navigatePathAsync?: AsyncPathNavigator + ) { this.parallelNameToId = new Map() for (const block of workflow.blocks) { if (workflow.parallels?.[block.id] && block.metadata?.name) { @@ -44,6 +54,27 @@ export class ParallelResolver implements Resolver { } resolve(reference: string, context: ResolutionContext): any { + return this.resolveInternal(reference, context, false) + } + + async resolveAsync(reference: string, context: ResolutionContext): Promise { + if (!this.navigatePathAsync) { + return this.resolve(reference, context) + } + return this.resolveInternal(reference, context, true) + } + + private async resolveInternal( + reference: string, + context: ResolutionContext, + useAsyncPath: true + ): Promise + private resolveInternal(reference: string, context: ResolutionContext, useAsyncPath: false): any + private resolveInternal( + reference: string, + context: ResolutionContext, + useAsyncPath: boolean + ): any | Promise { const parts = parseReferencePath(reference) if (parts.length === 0) { logger.warn('Invalid parallel reference', { reference }) @@ -74,8 +105,17 @@ export class ParallelResolver implements Resolver { ) } - if (rest.length > 0 && ParallelResolver.OUTPUT_PROPERTIES.has(rest[0])) { - return this.resolveOutput(targetParallelId, rest.slice(1), context) + if (rest.length > 0) { + const { property, pathParts: bracketPathParts } = splitLeadingBracketPath(rest[0]) + if (ParallelResolver.OUTPUT_PROPERTIES.has(property)) { + return useAsyncPath + ? this.resolveOutputAsync( + targetParallelId, + [...bracketPathParts, ...rest.slice(1)], + context + ) + : this.resolveOutput(targetParallelId, [...bracketPathParts, ...rest.slice(1)], context) + } } // Look up config using the original (non-cloned) ID @@ -86,18 +126,14 @@ export class ParallelResolver implements Resolver { return undefined } - if (!isGenericRef) { - if (!this.isBlockInParallelOrDescendant(context.currentNodeId, originalParallelId)) { - logger.warn('Block is not inside the referenced parallel', { - reference, - blockId: context.currentNodeId, - parallelId: targetParallelId, - }) - return undefined - } + const isContextual = + isGenericRef || this.isBlockInParallelOrDescendant(context.currentNodeId, originalParallelId) + + if (rest.length > 0 && !isContextual) { + throw new InvalidFieldError(firstPart, rest[0], [...PARALLEL_OUTPUT_FIELDS]) } - const branchIndex = extractBranchIndex(context.currentNodeId) + const branchIndex = this.resolveBranchIndex(targetParallelId, context) if (branchIndex === null) { return undefined } @@ -116,15 +152,12 @@ export class ParallelResolver implements Resolver { return result } - const property = rest[0] - const pathParts = rest.slice(1) + const [rawProperty, ...remainingPathParts] = rest + const { property, pathParts: bracketPathParts } = splitLeadingBracketPath(rawProperty) + const pathParts = [...bracketPathParts, ...remainingPathParts] if (!ParallelResolver.KNOWN_PROPERTIES.has(property)) { - const isCollection = parallelConfig.parallelType === 'collection' - const availableFields = isCollection - ? ['index', 'currentItem', 'items', 'result'] - : ['index', 'result'] - throw new InvalidFieldError(firstPart, property, availableFields) + throw new InvalidFieldError(firstPart, rawProperty, this.getAvailableFields(parallelConfig)) } let value: unknown @@ -142,12 +175,28 @@ export class ParallelResolver implements Resolver { } if (pathParts.length > 0) { - return navigatePath(value, pathParts) + return useAsyncPath && this.navigatePathAsync + ? this.navigatePathAsync(value, pathParts, context) + : navigatePath(value, pathParts, { executionContext: context.executionContext }) } return value } + private resolveBranchIndex(targetParallelId: string, context: ResolutionContext): number | null { + const mapping = context.executionContext.parallelBlockMapping?.get(context.currentNodeId) + if (mapping?.parallelId === targetParallelId) { + return mapping.iterationIndex + } + + const outerBranchIndex = extractOuterBranchIndex(context.currentNodeId) + if (outerBranchIndex !== undefined) { + return outerBranchIndex + } + + return extractBranchIndex(context.currentNodeId) + } + private findInnermostParallelForBlock(blockId: string): string | undefined { const baseId = stripCloneSuffixes(blockId) const parallels = this.workflow.parallels @@ -234,7 +283,31 @@ export class ParallelResolver implements Resolver { } const value = (output as Record).results if (pathParts.length > 0) { - return navigatePath(value, pathParts) + return navigatePath(value, pathParts, { executionContext: context.executionContext }) + } + if (!context.allowLargeValueRefs) { + assertNoLargeValueRefs(value) + } + return value + } + + private async resolveOutputAsync( + parallelId: string, + pathParts: string[], + context: ResolutionContext + ): Promise { + const output = context.executionState.getBlockOutput(parallelId) + if (!output || typeof output !== 'object') { + return undefined + } + const value = (output as Record).results + if (pathParts.length > 0) { + return this.navigatePathAsync + ? this.navigatePathAsync(value, pathParts, context) + : navigatePath(value, pathParts, { executionContext: context.executionContext }) + } + if (!context.allowLargeValueRefs) { + assertNoLargeValueRefs(value) } return value } @@ -278,4 +351,10 @@ export class ParallelResolver implements Resolver { return [] } + + private getAvailableFields(parallelConfig: SerializedParallel): string[] { + return parallelConfig.parallelType === 'collection' + ? [...COLLECTION_PARALLEL_CONTEXT_FIELDS] + : [...PARALLEL_CONTEXT_FIELDS] + } } diff --git a/apps/sim/executor/variables/resolvers/reference-async.server.ts b/apps/sim/executor/variables/resolvers/reference-async.server.ts new file mode 100644 index 00000000000..78dca4a3712 --- /dev/null +++ b/apps/sim/executor/variables/resolvers/reference-async.server.ts @@ -0,0 +1,120 @@ +import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' +import { + assertNoLargeValueRefs, + getLargeValueMaterializationError, + isLargeValueRef, +} from '@/lib/execution/payloads/large-value-ref' +import { materializeLargeValueRef } from '@/lib/execution/payloads/store' +import { hydrateUserFileWithBase64 } from '@/lib/uploads/utils/user-file-base64.server' +import type { ResolutionContext } from '@/executor/variables/resolvers/reference' + +async function materializeLargeValueRefOrThrow( + value: unknown, + context: ResolutionContext +): Promise { + if (!isLargeValueRef(value)) { + return value + } + const materialized = await materializeLargeValueRef(value, { + workspaceId: context.executionContext.workspaceId, + workflowId: context.executionContext.workflowId, + executionId: context.executionContext.executionId, + largeValueExecutionIds: context.executionContext.largeValueExecutionIds, + allowLargeValueWorkflowScope: context.executionContext.allowLargeValueWorkflowScope, + userId: context.executionContext.userId, + }) + if (materialized === undefined) { + throw getLargeValueMaterializationError(value) + } + return materialized +} + +async function hydrateExplicitBase64( + file: unknown, + context: ResolutionContext +): Promise { + if (!isUserFileWithMetadata(file)) { + return undefined + } + const hydrated = await hydrateUserFileWithBase64(file, { + requestId: context.executionContext.metadata.requestId, + workspaceId: context.executionContext.workspaceId, + workflowId: context.executionContext.workflowId, + executionId: context.executionContext.executionId, + largeValueExecutionIds: context.executionContext.largeValueExecutionIds, + allowLargeValueWorkflowScope: context.executionContext.allowLargeValueWorkflowScope, + userId: context.executionContext.userId, + maxBytes: context.executionContext.base64MaxBytes, + }) + if (!hydrated.base64) { + throw new Error( + `Base64 content for ${file.name} is unavailable or exceeds the configured inline limit.` + ) + } + return hydrated.base64 +} + +/** + * Server-side path navigation used during execution. It can hydrate persisted + * large values and UserFile.base64 only when the requested path explicitly asks + * for base64. + */ +export async function navigatePathAsync( + obj: any, + path: string[], + context: ResolutionContext +): Promise { + let current = obj + for (const part of path) { + current = await materializeLargeValueRefOrThrow(current, context) + + if (current === null || current === undefined) { + return undefined + } + + if (part === 'base64') { + const base64 = await hydrateExplicitBase64(current, context) + if (base64 !== undefined) { + current = base64 + continue + } + } + + const arrayMatch = part.match(/^([^[]+)(\[.+)$/) + if (arrayMatch) { + const [, prop, bracketsPart] = arrayMatch + current = + typeof current === 'object' && current !== null + ? (current as Record)[prop] + : undefined + current = await materializeLargeValueRefOrThrow(current, context) + if (current === undefined || current === null) { + return undefined + } + + const indices = bracketsPart.match(/\[(\d+)\]/g) + if (indices) { + for (const indexMatch of indices) { + current = await materializeLargeValueRefOrThrow(current, context) + if (current === null || current === undefined) { + return undefined + } + const idx = Number.parseInt(indexMatch.slice(1, -1), 10) + current = Array.isArray(current) ? current[idx] : undefined + } + } + } else if (/^\d+$/.test(part)) { + const index = Number.parseInt(part, 10) + current = Array.isArray(current) ? current[index] : undefined + } else { + current = + typeof current === 'object' && current !== null + ? (current as Record)[part] + : undefined + } + } + if (!context.allowLargeValueRefs) { + assertNoLargeValueRefs(current) + } + return current +} diff --git a/apps/sim/executor/variables/resolvers/reference.ts b/apps/sim/executor/variables/resolvers/reference.ts index 35d32272739..70a49a4d11b 100644 --- a/apps/sim/executor/variables/resolvers/reference.ts +++ b/apps/sim/executor/variables/resolvers/reference.ts @@ -1,3 +1,5 @@ +import { materializeLargeValueRefSyncOrThrow } from '@/lib/execution/payloads/cache' +import { assertNoLargeValueRefs, isLargeValueRef } from '@/lib/execution/payloads/large-value-ref' import type { ExecutionState, LoopScope } from '@/executor/execution/state' import type { ExecutionContext } from '@/executor/types' export interface ResolutionContext { @@ -5,13 +7,21 @@ export interface ResolutionContext { executionState: ExecutionState currentNodeId: string loopScope?: LoopScope + allowLargeValueRefs?: boolean } export interface Resolver { canResolve(reference: string): boolean resolve(reference: string, context: ResolutionContext): any + resolveAsync?(reference: string, context: ResolutionContext): Promise } +export type AsyncPathNavigator = ( + obj: any, + path: string[], + context: ResolutionContext +) => Promise + /** * Sentinel value indicating a reference was resolved to a known block * that produced no output (e.g., the block exists in the workflow but @@ -20,6 +30,19 @@ export interface Resolver { */ export const RESOLVED_EMPTY = Symbol('RESOLVED_EMPTY') +export function splitLeadingBracketPath(part: string): { property: string; pathParts: string[] } { + const bracketMatch = part.match(/^([^[]+)((?:\[\d+\])+)$/) + if (!bracketMatch) { + return { property: part, pathParts: [] } + } + + const indices = bracketMatch[2].match(/\[(\d+)\]/g) ?? [] + return { + property: bracketMatch[1], + pathParts: indices.map((indexMatch) => indexMatch.slice(1, -1)), + } +} + /** * Navigate through nested object properties using a path array. * Supports dot notation and array indices. @@ -28,9 +51,17 @@ export const RESOLVED_EMPTY = Symbol('RESOLVED_EMPTY') * navigatePath({a: {b: {c: 1}}}, ['a', 'b', 'c']) => 1 * navigatePath({items: [{name: 'test'}]}, ['items', '0', 'name']) => 'test' */ -export function navigatePath(obj: any, path: string[]): any { +export function navigatePath( + obj: any, + path: string[], + options: { allowLargeValueRefs?: boolean; executionContext?: ExecutionContext } = {} +): any { let current = obj for (const part of path) { + if (isLargeValueRef(current)) { + current = materializeLargeValueRefSyncOrThrow(current, options.executionContext) + } + if (current === null || current === undefined) { return undefined } @@ -42,6 +73,9 @@ export function navigatePath(obj: any, path: string[]): any { typeof current === 'object' && current !== null ? (current as Record)[prop] : undefined + if (isLargeValueRef(current)) { + current = materializeLargeValueRefSyncOrThrow(current, options.executionContext) + } if (current === undefined || current === null) { return undefined } @@ -52,6 +86,9 @@ export function navigatePath(obj: any, path: string[]): any { if (current === null || current === undefined) { return undefined } + if (isLargeValueRef(current)) { + current = materializeLargeValueRefSyncOrThrow(current, options.executionContext) + } const idx = Number.parseInt(indexMatch.slice(1, -1), 10) current = Array.isArray(current) ? current[idx] : undefined } @@ -66,5 +103,8 @@ export function navigatePath(obj: any, path: string[]): any { : undefined } } + if (!options.allowLargeValueRefs) { + assertNoLargeValueRefs(current) + } return current } diff --git a/apps/sim/executor/variables/resolvers/workflow.ts b/apps/sim/executor/variables/resolvers/workflow.ts index f11612e2ee2..ad2c667949e 100644 --- a/apps/sim/executor/variables/resolvers/workflow.ts +++ b/apps/sim/executor/variables/resolvers/workflow.ts @@ -57,7 +57,7 @@ export class WorkflowResolver implements Resolver { // If there are additional path parts, navigate deeper if (pathParts.length > 0) { - return navigatePath(value, pathParts) + return navigatePath(value, pathParts, { executionContext: context.executionContext }) } return value diff --git a/apps/sim/hooks/queries/tasks.test.ts b/apps/sim/hooks/queries/tasks.test.ts index 0edae3bb93b..39b2c88b7e8 100644 --- a/apps/sim/hooks/queries/tasks.test.ts +++ b/apps/sim/hooks/queries/tasks.test.ts @@ -100,7 +100,7 @@ describe('tasks query boundary parsing', () => { id: 'chat-1', title: 'Task history', messages: [], - conversationId: 'stream-1', + activeStreamId: 'stream-1', resources: [{ type: 'file', id: 'file-1', title: 'Spec.md' }], streamSnapshot: { events: [], @@ -144,7 +144,7 @@ describe('tasks query boundary parsing', () => { ) await expect(fetchChatHistory('chat-1')).rejects.toThrow( - 'Invalid copilot chat response: chat.resources[0].type is invalid' + 'Invalid chat response: chat.resources[0].type is invalid' ) }) diff --git a/apps/sim/hooks/queries/tasks.ts b/apps/sim/hooks/queries/tasks.ts index edba5ff6f1e..d167270f922 100644 --- a/apps/sim/hooks/queries/tasks.ts +++ b/apps/sim/hooks/queries/tasks.ts @@ -57,8 +57,6 @@ export const taskKeys = { detail: (chatId: string | undefined) => [...taskKeys.details(), chatId ?? ''] as const, } -type ChatHistorySource = 'copilot' | 'mothership' - function isRecord(value: unknown): value is Record { return Boolean(value) && typeof value === 'object' && !Array.isArray(value) } @@ -150,30 +148,28 @@ function parseStrictStreamSnapshot( return snapshot } -function parseChatHistory(value: unknown, source: ChatHistorySource): TaskChatHistory { - const responseContext = `Invalid ${source} chat response` +function parseChatHistory(value: unknown): TaskChatHistory { + const responseContext = 'Invalid chat response' const chatContext = `${responseContext}: chat` assertValid(isRecord(value), `${responseContext}: body must be an object`) assertValid(isRecord(value.chat), `${chatContext} must be an object`) const chat = value.chat - const activeStreamField = source === 'mothership' ? 'conversationId' : 'activeStreamId' - const activeStreamId = chat[activeStreamField] assertValid(typeof chat.id === 'string', `${chatContext}.id must be a string`) assertValid(isNullableString(chat.title), `${chatContext}.title must be a string or null`) assertValid(Array.isArray(chat.messages), `${chatContext}.messages must be an array`) assertValid( - isNullableString(activeStreamId), - `${chatContext}.${activeStreamField} must be a string or null` + isNullableString(chat.activeStreamId), + `${chatContext}.activeStreamId must be a string or null` ) return { id: chat.id, title: chat.title, messages: normalizeMessages(chat.messages), - activeStreamId, + activeStreamId: chat.activeStreamId, resources: parseResources(chat.resources, `${chatContext}.resources`), streamSnapshot: parseStrictStreamSnapshot(chat.streamSnapshot, `${chatContext}.streamSnapshot`), } @@ -233,7 +229,7 @@ export async function fetchChatHistory( params: { chatId }, signal, }) - return parseChatHistory(data, 'mothership') + return parseChatHistory(data) } catch (error) { if (!isApiClientError(error)) throw error // Fall through to the legacy copilot-shape alias on any HTTP error (typically 404 @@ -251,7 +247,7 @@ export async function fetchChatHistory( throw new Error('Failed to load chat') } - return parseChatHistory(await copilotRes.json(), 'copilot') + return parseChatHistory(await copilotRes.json()) } /** diff --git a/apps/sim/hooks/use-collaborative-workflow.ts b/apps/sim/hooks/use-collaborative-workflow.ts index 10585e1f8a9..a4b58338886 100644 --- a/apps/sim/hooks/use-collaborative-workflow.ts +++ b/apps/sim/hooks/use-collaborative-workflow.ts @@ -337,6 +337,9 @@ export function useCollaborativeWorkflow() { if (config.count !== undefined) { useWorkflowStore.getState().updateParallelCount(payload.id, config.count) } + if (config.batchSize !== undefined) { + useWorkflowStore.getState().updateParallelBatchSize(payload.id, config.batchSize) + } if (config.distribution !== undefined) { useWorkflowStore .getState() @@ -1728,6 +1731,7 @@ export function useCollaborativeWorkflow() { let newCount = currentBlock.data?.count || 5 let newDistribution = currentBlock.data?.collection || '' + const batchSize = currentBlock.data?.batchSize || 20 if (parallelType === 'count') { newDistribution = '' @@ -1742,6 +1746,7 @@ export function useCollaborativeWorkflow() { count: newCount, distribution: newDistribution, parallelType, + batchSize, } executeQueuedOperation( @@ -1752,6 +1757,7 @@ export function useCollaborativeWorkflow() { useWorkflowStore.getState().updateParallelType(parallelId, parallelType) useWorkflowStore.getState().updateParallelCount(parallelId, newCount) useWorkflowStore.getState().updateParallelCollection(parallelId, newDistribution) + useWorkflowStore.getState().updateParallelBatchSize(parallelId, batchSize) } ) }, @@ -1768,41 +1774,52 @@ export function useCollaborativeWorkflow() { .filter((b) => b.data?.parentId === nodeId) .map((b) => b.id) + const clampedCount = Math.max(1, count) + if (iterationType === 'loop') { const currentLoopType = currentBlock.data?.loopType || 'for' - const currentCollection = currentBlock.data?.collection || '' + const existingLoop = useWorkflowStore.getState().loops[nodeId] + const nextForEachItems = existingLoop?.forEachItems ?? currentBlock.data?.collection ?? '' + const nextWhileCondition = + existingLoop?.whileCondition ?? currentBlock.data?.whileCondition ?? '' + const nextDoWhileCondition = + existingLoop?.doWhileCondition ?? currentBlock.data?.doWhileCondition ?? '' const config = { id: nodeId, nodes: childNodes, - iterations: Math.max(1, Math.min(1000, count)), // Clamp between 1-1000 for loops + iterations: clampedCount, loopType: currentLoopType, - forEachItems: currentCollection, + forEachItems: nextForEachItems, + whileCondition: nextWhileCondition, + doWhileCondition: nextDoWhileCondition, } executeQueuedOperation( SUBFLOW_OPERATIONS.UPDATE, OPERATION_TARGETS.SUBFLOW, { id: nodeId, type: 'loop', config }, - () => useWorkflowStore.getState().updateLoopCount(nodeId, count) + () => useWorkflowStore.getState().updateLoopCount(nodeId, clampedCount) ) } else { const currentDistribution = currentBlock.data?.collection || '' const currentParallelType = currentBlock.data?.parallelType || 'count' + const batchSize = currentBlock.data?.batchSize || 20 const config = { id: nodeId, nodes: childNodes, - count: Math.max(1, Math.min(20, count)), // Clamp between 1-20 for parallels + count: clampedCount, distribution: currentDistribution, parallelType: currentParallelType, + batchSize, } executeQueuedOperation( SUBFLOW_OPERATIONS.UPDATE, OPERATION_TARGETS.SUBFLOW, { id: nodeId, type: 'parallel', config }, - () => useWorkflowStore.getState().updateParallelCount(nodeId, count) + () => useWorkflowStore.getState().updateParallelCount(nodeId, clampedCount) ) } }, @@ -1860,6 +1877,7 @@ export function useCollaborativeWorkflow() { } else { const currentCount = currentBlock.data?.count || 5 const currentParallelType = currentBlock.data?.parallelType || 'count' + const batchSize = currentBlock.data?.batchSize || 20 const config = { id: nodeId, @@ -1867,6 +1885,7 @@ export function useCollaborativeWorkflow() { count: currentCount, distribution: collection, parallelType: currentParallelType, + batchSize, } executeQueuedOperation( @@ -1880,6 +1899,38 @@ export function useCollaborativeWorkflow() { [executeQueuedOperation] ) + const collaborativeUpdateParallelBatchSize = useCallback( + (parallelId: string, batchSize: number) => { + const currentBlock = useWorkflowStore.getState().blocks[parallelId] + if (!currentBlock || currentBlock.type !== 'parallel') return + + const childNodes = Object.values(useWorkflowStore.getState().blocks) + .filter((b) => b.data?.parentId === parallelId) + .map((b) => b.id) + const currentCount = currentBlock.data?.count || 5 + const currentDistribution = currentBlock.data?.collection || '' + const currentParallelType = currentBlock.data?.parallelType || 'count' + const clampedBatchSize = Math.max(1, Math.min(20, batchSize)) + + const config = { + id: parallelId, + nodes: childNodes, + count: currentCount, + distribution: currentDistribution, + parallelType: currentParallelType, + batchSize: clampedBatchSize, + } + + executeQueuedOperation( + SUBFLOW_OPERATIONS.UPDATE, + OPERATION_TARGETS.SUBFLOW, + { id: parallelId, type: 'parallel', config }, + () => useWorkflowStore.getState().updateParallelBatchSize(parallelId, clampedBatchSize) + ) + }, + [executeQueuedOperation] + ) + const collaborativeUpdateVariable = useCallback( (variableId: string, field: 'name' | 'value' | 'type', value: any) => { executeQueuedOperation( @@ -2137,6 +2188,7 @@ export function useCollaborativeWorkflow() { // Collaborative loop/parallel operations collaborativeUpdateLoopType, collaborativeUpdateParallelType, + collaborativeUpdateParallelBatchSize, // Unified iteration operations collaborativeUpdateIterationCount, diff --git a/apps/sim/hooks/use-undo-redo.ts b/apps/sim/hooks/use-undo-redo.ts index 86fe2a6bcce..025c087de0e 100644 --- a/apps/sim/hooks/use-undo-redo.ts +++ b/apps/sim/hooks/use-undo-redo.ts @@ -617,7 +617,9 @@ export function useUndoRedo() { const currentCount = currentBlock.data?.count || 5 const currentParallelType = currentBlock.data?.parallelType || 'count' const currentDistribution = currentBlock.data?.collection || '' + const currentBatchSize = currentBlock.data?.batchSize || 20 const nextCount = Number.parseInt(String(update.after), 10) + const nextBatchSize = Number.parseInt(String(update.after), 10) const config = { id: update.blockId, nodes: childNodes, @@ -630,6 +632,10 @@ export function useUndoRedo() { ? update.after : currentDistribution, parallelType: currentParallelType, + batchSize: + update.fieldId === WORKFLOW_SEARCH_SUBFLOW_FIELD_IDS.batchSize + ? nextBatchSize + : currentBatchSize, } addToQueue({ @@ -650,6 +656,13 @@ export function useUndoRedo() { return } + if (update.fieldId === WORKFLOW_SEARCH_SUBFLOW_FIELD_IDS.batchSize) { + if (!Number.isNaN(nextBatchSize)) { + useWorkflowStore.getState().updateParallelBatchSize(update.blockId, nextBatchSize) + } + return + } + useWorkflowStore.getState().updateParallelCollection(update.blockId, String(update.after)) }, [activeWorkflowId, addToQueue, userId] diff --git a/apps/sim/lib/a2a/utils.ts b/apps/sim/lib/a2a/utils.ts index a1e8f79a655..d89a8cec040 100644 --- a/apps/sim/lib/a2a/utils.ts +++ b/apps/sim/lib/a2a/utils.ts @@ -14,11 +14,17 @@ import { type Client, ClientFactory, ClientFactoryOptions, + DefaultAgentCardResolver, + JsonRpcTransportFactory, + RestTransportFactory, } from '@a2a-js/sdk/client' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' -import { validateUrlWithDNS } from '@/lib/core/security/input-validation.server' +import { + secureFetchWithPinnedIP, + validateUrlWithDNS, +} from '@/lib/core/security/input-validation.server' import { isInternalFileUrl } from '@/lib/uploads/utils/file-utils' import { A2A_TERMINAL_STATES } from './constants' @@ -60,13 +66,76 @@ export async function createA2AClient(agentUrl: string, apiKey?: string): Promis throw new Error(validation.error || 'Agent URL validation failed') } + const resolvedIP = validation.resolvedIP! + + const pinnedFetch = async ( + input: Parameters[0], + init?: Parameters[1] + ): Promise => { + const url = input instanceof Request ? input.url : input.toString() + const method = init?.method ?? (input instanceof Request ? input.method : undefined) + + const rawHeaders = init?.headers ?? (input instanceof Request ? input.headers : undefined) + const headers = + rawHeaders instanceof Headers + ? Object.fromEntries(rawHeaders.entries()) + : Array.isArray(rawHeaders) + ? Object.fromEntries(rawHeaders as string[][]) + : (rawHeaders as Record | undefined) + + let body: string | Buffer | Uint8Array | undefined + if (init?.body !== undefined && init.body !== null) { + if (typeof init.body === 'string' || Buffer.isBuffer(init.body)) { + body = init.body as string | Buffer + } else if (init.body instanceof Uint8Array) { + body = init.body + } else if (init.body instanceof ArrayBuffer) { + body = new Uint8Array(init.body) + } else { + const text = await new Response(init.body as BodyInit).text() + if (text) body = text + } + } else if (init?.body === undefined && input instanceof Request && !input.bodyUsed) { + const text = await input.text() + if (text) body = text + } + + const signal = + init?.signal instanceof AbortSignal + ? init.signal + : input instanceof Request && input.signal instanceof AbortSignal + ? input.signal + : undefined + + const res = await secureFetchWithPinnedIP(url, resolvedIP, { method, headers, body, signal }) + const resHeaders = new Headers(res.headers.toRecord()) + return new Response(res.body, { + status: res.status, + statusText: res.statusText, + headers: resHeaders, + }) + } + + const pinnedTransports = [ + new JsonRpcTransportFactory({ fetchImpl: pinnedFetch }), + new RestTransportFactory({ fetchImpl: pinnedFetch }), + ] + + const pinnedCardResolver = new DefaultAgentCardResolver({ fetchImpl: pinnedFetch }) + + const baseOptions = ClientFactoryOptions.createFrom(ClientFactoryOptions.default, { + transports: pinnedTransports, + cardResolver: pinnedCardResolver, + }) + const factoryOptions = apiKey - ? ClientFactoryOptions.createFrom(ClientFactoryOptions.default, { + ? ClientFactoryOptions.createFrom(baseOptions, { clientConfig: { interceptors: [new ApiKeyInterceptor(apiKey)], }, }) - : ClientFactoryOptions.default + : baseOptions + const factory = new ClientFactory(factoryOptions) // Try standard A2A path first (/.well-known/agent.json) diff --git a/apps/sim/lib/api/contracts/execution-payloads.ts b/apps/sim/lib/api/contracts/execution-payloads.ts new file mode 100644 index 00000000000..485918dc4ac --- /dev/null +++ b/apps/sim/lib/api/contracts/execution-payloads.ts @@ -0,0 +1,31 @@ +import { z } from 'zod' +import { + isLargeValueStorageKey, + LARGE_VALUE_KINDS, + LARGE_VALUE_REF_MARKER, + LARGE_VALUE_REF_VERSION, +} from '@/lib/execution/payloads/large-value-ref' + +export const largeValueRefSchema = z + .object({ + [LARGE_VALUE_REF_MARKER]: z.literal(true), + version: z.literal(LARGE_VALUE_REF_VERSION), + id: z.string().regex(/^lv_[A-Za-z0-9_-]{12}$/, 'Invalid large value reference ID'), + kind: z.enum(LARGE_VALUE_KINDS), + size: z.number().int().positive(), + key: z.string().optional(), + executionId: z.string().optional(), + preview: z.unknown().optional(), + }) + .strict() + .superRefine((value, ctx) => { + if (value.key && !isLargeValueStorageKey(value.key, value.id, value.executionId)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['key'], + message: 'Large value reference key must point to execution-scoped server storage', + }) + } + }) + +export type LargeValueRefResponse = z.output diff --git a/apps/sim/lib/api/contracts/hotspots.ts b/apps/sim/lib/api/contracts/hotspots.ts index 099170bc8be..4a3b92d371d 100644 --- a/apps/sim/lib/api/contracts/hotspots.ts +++ b/apps/sim/lib/api/contracts/hotspots.ts @@ -102,6 +102,9 @@ export const functionExecuteContract = defineRouteContract({ workflowVariables: unknownRecordSchema.optional().default({}), contextVariables: unknownRecordSchema.optional().default({}), workflowId: z.string().optional(), + executionId: z.string().optional(), + largeValueExecutionIds: z.array(z.string()).optional(), + allowLargeValueWorkflowScope: z.boolean().optional(), workspaceId: z.string().optional(), userId: z.string().optional(), isCustomTool: z.boolean().optional().default(false), diff --git a/apps/sim/lib/api/contracts/index.ts b/apps/sim/lib/api/contracts/index.ts index 4ee2d4533ee..4c9858af67c 100644 --- a/apps/sim/lib/api/contracts/index.ts +++ b/apps/sim/lib/api/contracts/index.ts @@ -11,6 +11,7 @@ export * from './credential-sets' export * from './credentials' export * from './demo-requests' export * from './environment' +export * from './execution-payloads' export * from './file-uploads' export * from './folders' export * from './hotspots' diff --git a/apps/sim/lib/api/contracts/mothership-tasks.ts b/apps/sim/lib/api/contracts/mothership-tasks.ts index fe8c0a182b7..5be16920551 100644 --- a/apps/sim/lib/api/contracts/mothership-tasks.ts +++ b/apps/sim/lib/api/contracts/mothership-tasks.ts @@ -256,6 +256,7 @@ export const mothershipExecuteResponseSchema = z .object({ content: z.string().optional(), model: z.literal('mothership'), + conversationId: z.string(), tokens: z .object({ prompt: z.number().optional(), @@ -283,7 +284,7 @@ export const getMothershipChatResponseSchema = z.object({ id: z.string(), title: z.string().nullable(), messages: z.array(z.unknown()), - conversationId: z.string().nullable(), + activeStreamId: z.string().nullable(), resources: z.array(z.unknown()), createdAt: z.union([z.string(), z.date()]).nullable().optional(), updatedAt: z.union([z.string(), z.date()]).nullable().optional(), diff --git a/apps/sim/lib/api/contracts/workflows.ts b/apps/sim/lib/api/contracts/workflows.ts index af55e5ef708..46e5095c933 100644 --- a/apps/sim/lib/api/contracts/workflows.ts +++ b/apps/sim/lib/api/contracts/workflows.ts @@ -20,6 +20,7 @@ const workflowBlockDataSchema = z.object({ whileCondition: z.string().optional(), doWhileCondition: z.string().optional(), parallelType: z.enum(['collection', 'count']).optional(), + batchSize: z.number().optional(), type: z.string().optional(), canonicalModes: z.record(z.string(), z.enum(['basic', 'advanced'])).optional(), }) @@ -90,6 +91,7 @@ const workflowParallelSchema = z.object({ .optional(), count: z.number().optional(), parallelType: z.enum(['count', 'collection']).optional(), + batchSize: z.number().optional(), enabled: z.boolean().optional(), locked: z.boolean().optional(), }) diff --git a/apps/sim/lib/auth/credential-access.ts b/apps/sim/lib/auth/credential-access.ts index 05e017c87a3..97a9bf50b15 100644 --- a/apps/sim/lib/auth/credential-access.ts +++ b/apps/sim/lib/auth/credential-access.ts @@ -13,6 +13,7 @@ export interface CredentialAccessResult { credentialOwnerUserId?: string workspaceId?: string resolvedCredentialId?: string + credentialType?: 'oauth' | 'service_account' } /** @@ -114,6 +115,7 @@ export async function authorizeCredentialUse( credentialOwnerUserId: actingUserId, workspaceId: platformCredential.workspaceId, resolvedCredentialId: platformCredential.id, + credentialType: 'service_account', } } @@ -182,6 +184,7 @@ export async function authorizeCredentialUse( credentialOwnerUserId: accountRow.userId, workspaceId: platformCredential.workspaceId, resolvedCredentialId: platformCredential.accountId, + credentialType: 'oauth', } } @@ -252,6 +255,7 @@ export async function authorizeCredentialUse( credentialOwnerUserId: accountRow.userId, workspaceId: workflowContext.workspaceId, resolvedCredentialId: workspaceCredential.accountId, + credentialType: 'oauth', } } @@ -279,5 +283,6 @@ export async function authorizeCredentialUse( requesterUserId: auth.userId, credentialOwnerUserId: legacyAccount.userId, resolvedCredentialId: credentialId, + credentialType: 'oauth', } } diff --git a/apps/sim/lib/copilot/chat/stream-liveness.test.ts b/apps/sim/lib/copilot/chat/stream-liveness.test.ts new file mode 100644 index 00000000000..f7294b4c6ec --- /dev/null +++ b/apps/sim/lib/copilot/chat/stream-liveness.test.ts @@ -0,0 +1,154 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockAnd, mockEq, mockGetChatStreamLockOwners, mockSet, mockUpdate, mockWhere } = vi.hoisted( + () => ({ + mockAnd: vi.fn((...conditions: unknown[]) => ({ type: 'and', conditions })), + mockEq: vi.fn((field: unknown, value: unknown) => ({ type: 'eq', field, value })), + mockGetChatStreamLockOwners: vi.fn(), + mockSet: vi.fn(), + mockUpdate: vi.fn(), + mockWhere: vi.fn(), + }) +) + +vi.mock('@sim/db', () => ({ + db: { update: mockUpdate }, +})) + +vi.mock('@sim/db/schema', () => ({ + copilotChats: { + id: 'copilotChats.id', + conversationId: 'copilotChats.conversationId', + }, +})) + +vi.mock('drizzle-orm', () => ({ + and: mockAnd, + eq: mockEq, +})) + +vi.mock('@/lib/copilot/request/session', () => ({ + getChatStreamLockOwners: mockGetChatStreamLockOwners, +})) + +import { reconcileChatStreamMarkers } from '@/lib/copilot/chat/stream-liveness' + +describe('reconcileChatStreamMarkers', () => { + beforeEach(() => { + vi.clearAllMocks() + mockSet.mockReturnValue({ where: mockWhere }) + mockUpdate.mockReturnValue({ set: mockSet }) + mockWhere.mockResolvedValue(undefined) + mockGetChatStreamLockOwners.mockResolvedValue({ + status: 'verified', + ownersByChatId: new Map(), + }) + }) + + it('clears a persisted stream marker when Redis verifies no lock owner exists', async () => { + const markers = await reconcileChatStreamMarkers([ + { chatId: 'chat-stuck', streamId: 'stream-orphaned' }, + ]) + + expect(mockGetChatStreamLockOwners).toHaveBeenCalledWith(['chat-stuck']) + expect(markers.get('chat-stuck')).toEqual({ + chatId: 'chat-stuck', + streamId: null, + status: 'inactive', + }) + }) + + it('repairs a verified stale persisted stream marker when requested', async () => { + await reconcileChatStreamMarkers([{ chatId: 'chat-stuck', streamId: 'stream-orphaned' }], { + repairVerifiedStaleMarkers: true, + }) + + expect(mockUpdate).toHaveBeenCalled() + expect(mockSet).toHaveBeenCalledWith({ conversationId: null }) + expect(mockWhere).toHaveBeenCalledWith( + mockAnd( + mockEq('copilotChats.id', 'chat-stuck'), + mockEq('copilotChats.conversationId', 'stream-orphaned') + ) + ) + }) + + it('uses the canonical Redis owner when the persisted stream marker is stale', async () => { + mockGetChatStreamLockOwners.mockResolvedValueOnce({ + status: 'verified', + ownersByChatId: new Map([['chat-mismatch', 'stream-live']]), + }) + + const markers = await reconcileChatStreamMarkers([ + { chatId: 'chat-mismatch', streamId: 'stream-stale' }, + ]) + + expect(markers.get('chat-mismatch')).toEqual({ + chatId: 'chat-mismatch', + streamId: 'stream-live', + status: 'active', + }) + }) + + it('preserves persisted stream markers when Redis state is unknown', async () => { + mockGetChatStreamLockOwners.mockResolvedValueOnce({ + status: 'unknown', + ownersByChatId: new Map(), + }) + + const markers = await reconcileChatStreamMarkers([ + { chatId: 'chat-remote', streamId: 'stream-remote' }, + ]) + + expect(markers.get('chat-remote')).toEqual({ + chatId: 'chat-remote', + streamId: 'stream-remote', + status: 'unknown', + }) + }) + + it('preserves a persisted marker when unknown local owner differs', async () => { + mockGetChatStreamLockOwners.mockResolvedValueOnce({ + status: 'unknown', + ownersByChatId: new Map([['chat-mismatch', 'stream-local']]), + }) + + const markers = await reconcileChatStreamMarkers([ + { chatId: 'chat-mismatch', streamId: 'stream-persisted' }, + ]) + + expect(markers.get('chat-mismatch')).toEqual({ + chatId: 'chat-mismatch', + streamId: 'stream-persisted', + status: 'unknown', + }) + }) + + it('treats a null persisted marker as inactive even when Redis still holds a lock (post-completion teardown window)', async () => { + mockGetChatStreamLockOwners.mockResolvedValueOnce({ + status: 'verified', + ownersByChatId: new Map([['chat-starting', 'stream-starting']]), + }) + + const markers = await reconcileChatStreamMarkers([{ chatId: 'chat-starting', streamId: null }]) + + expect(markers.get('chat-starting')).toEqual({ + chatId: 'chat-starting', + streamId: null, + status: 'inactive', + }) + }) + + it('does not query locks when no chats have persisted stream markers', async () => { + const markers = await reconcileChatStreamMarkers([{ chatId: 'chat-idle', streamId: null }]) + + expect(markers.get('chat-idle')).toEqual({ + chatId: 'chat-idle', + streamId: null, + status: 'inactive', + }) + }) +}) diff --git a/apps/sim/lib/copilot/chat/stream-liveness.ts b/apps/sim/lib/copilot/chat/stream-liveness.ts new file mode 100644 index 00000000000..82a92acbd24 --- /dev/null +++ b/apps/sim/lib/copilot/chat/stream-liveness.ts @@ -0,0 +1,135 @@ +import { db } from '@sim/db' +import { copilotChats } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { and, eq } from 'drizzle-orm' +import { getChatStreamLockOwners } from '@/lib/copilot/request/session' + +const logger = createLogger('ChatStreamLiveness') + +export interface ChatStreamMarkerCandidate { + chatId: string + streamId: string | null +} + +export interface ReconciledChatStreamMarker { + chatId: string + streamId: string | null + status: 'active' | 'inactive' | 'unknown' +} + +interface ReconcileChatStreamMarkersOptions { + repairVerifiedStaleMarkers?: boolean +} + +/** + * Reconciles persisted chat stream markers against the runtime stream lock. + * + * Redis lock ownership is the canonical live-stream signal. When the lookup is + * verified, missing owners clear stale persisted markers and present owners win + * over stale DB values. When Redis state is unknown, persisted markers are + * preserved so a transient Redis failure in a multi-pod deployment does not + * incorrectly hide a live stream owned by another pod. + */ +export async function reconcileChatStreamMarkers( + candidates: ChatStreamMarkerCandidate[], + options: ReconcileChatStreamMarkersOptions = {} +): Promise> { + const results = new Map() + + for (const candidate of candidates) { + if (candidate.streamId === null) { + results.set(candidate.chatId, { + chatId: candidate.chatId, + streamId: null, + status: 'inactive', + }) + continue + } + results.set(candidate.chatId, { + chatId: candidate.chatId, + streamId: candidate.streamId, + status: 'unknown', + }) + } + + const candidatesWithMarkers = candidates.filter((candidate) => candidate.streamId !== null) + if (candidatesWithMarkers.length === 0) { + return results + } + + const { status, ownersByChatId } = await getChatStreamLockOwners( + candidatesWithMarkers.map((candidate) => candidate.chatId) + ) + + for (const candidate of candidatesWithMarkers) { + const owner = ownersByChatId.get(candidate.chatId) + if (owner && (status === 'verified' || owner === candidate.streamId)) { + results.set(candidate.chatId, { + chatId: candidate.chatId, + streamId: owner, + status: 'active', + }) + continue + } + + if (status === 'verified') { + results.set(candidate.chatId, { + chatId: candidate.chatId, + streamId: null, + status: 'inactive', + }) + continue + } + + results.set(candidate.chatId, { + chatId: candidate.chatId, + streamId: candidate.streamId, + status: 'unknown', + }) + } + + if (options.repairVerifiedStaleMarkers) { + await repairVerifiedStaleMarkers(candidates, results) + } + + return results +} + +async function repairVerifiedStaleMarkers( + candidates: ChatStreamMarkerCandidate[], + results: Map +): Promise { + const staleCandidates = candidates.filter( + (candidate): candidate is { chatId: string; streamId: string } => { + const result = results.get(candidate.chatId) + return ( + candidate.streamId !== null && result?.status === 'inactive' && result.streamId === null + ) + } + ) + + if (staleCandidates.length === 0) return + + await Promise.all( + staleCandidates.map(async (candidate) => { + try { + await db + .update(copilotChats) + .set({ conversationId: null }) + .where( + and( + eq(copilotChats.id, candidate.chatId), + eq(copilotChats.conversationId, candidate.streamId) + ) + ) + } catch (error) { + logger.warn('Failed to repair stale chat stream marker', { + chatId: candidate.chatId, + streamId: candidate.streamId, + error: toError(error).message, + }) + } + }) + ) +} diff --git a/apps/sim/lib/copilot/request/session/abort.test.ts b/apps/sim/lib/copilot/request/session/abort.test.ts index bdfd5d39cbb..9e8e2feae82 100644 --- a/apps/sim/lib/copilot/request/session/abort.test.ts +++ b/apps/sim/lib/copilot/request/session/abort.test.ts @@ -22,7 +22,12 @@ vi.mock('@/lib/copilot/request/otel', () => ({ fn({ setAttribute: vi.fn() }), })) -import { startAbortPoller } from '@/lib/copilot/request/session/abort' +import { + acquirePendingChatStream, + getChatStreamLockOwners, + releasePendingChatStream, + startAbortPoller, +} from '@/lib/copilot/request/session/abort' describe('startAbortPoller heartbeat', () => { beforeEach(() => { @@ -159,3 +164,93 @@ describe('startAbortPoller heartbeat', () => { } }) }) + +describe('getChatStreamLockOwners', () => { + beforeEach(() => { + vi.clearAllMocks() + redisConfigMockFns.mockGetRedisClient.mockReturnValue(null) + }) + + it('returns a verified empty owner map when no chat ids are provided', async () => { + const result = await getChatStreamLockOwners([]) + expect(result.status).toBe('verified') + expect(result.ownersByChatId.size).toBe(0) + }) + + it('returns Redis lock owners keyed by chat id', async () => { + const mget = vi.fn().mockResolvedValue(['stream-1', null, 'stream-3']) + redisConfigMockFns.mockGetRedisClient.mockReturnValue({ mget } as never) + + const result = await getChatStreamLockOwners(['chat-1', 'chat-2', 'chat-3']) + + expect(mget).toHaveBeenCalledWith([ + 'copilot:chat-stream-lock:chat-1', + 'copilot:chat-stream-lock:chat-2', + 'copilot:chat-stream-lock:chat-3', + ]) + expect(result.status).toBe('verified') + expect(result.ownersByChatId).toEqual( + new Map([ + ['chat-1', 'stream-1'], + ['chat-3', 'stream-3'], + ]) + ) + }) + + it('returns a verified empty map when every lock has expired in Redis', async () => { + const mget = vi.fn().mockResolvedValue([null, null]) + redisConfigMockFns.mockGetRedisClient.mockReturnValue({ mget } as never) + + const result = await getChatStreamLockOwners(['chat-stuck-1', 'chat-stuck-2']) + + expect(result.status).toBe('verified') + expect(result.ownersByChatId.size).toBe(0) + }) + + it('trusts verified Redis null over a process-local pending stream', async () => { + const mget = vi.fn().mockResolvedValue([null]) + redisConfigMockFns.mockGetRedisClient.mockReturnValue({ mget } as never) + await acquirePendingChatStream('chat-local', 'stream-local') + + try { + const result = await getChatStreamLockOwners(['chat-local']) + + expect(result.status).toBe('verified') + expect(result.ownersByChatId.size).toBe(0) + } finally { + await releasePendingChatStream('chat-local', 'stream-local') + } + }) + + it('returns unknown status when Redis is unavailable', async () => { + redisConfigMockFns.mockGetRedisClient.mockReturnValue(null) + + const result = await getChatStreamLockOwners(['chat-1', 'chat-2']) + + expect(result.status).toBe('unknown') + expect(result.ownersByChatId.size).toBe(0) + }) + + it('preserves local pending stream owners when Redis is unavailable', async () => { + await acquirePendingChatStream('chat-local', 'stream-local') + + try { + const result = await getChatStreamLockOwners(['chat-local', 'chat-remote']) + + expect(result.status).toBe('unknown') + expect(result.ownersByChatId).toEqual(new Map([['chat-local', 'stream-local']])) + } finally { + await releasePendingChatStream('chat-local', 'stream-local') + } + }) + + it('returns unknown status without throwing when mget rejects', async () => { + const mget = vi.fn().mockRejectedValue(new Error('redis down')) + redisConfigMockFns.mockGetRedisClient.mockReturnValue({ mget } as never) + + const result = await getChatStreamLockOwners(['chat-1', 'chat-2']) + + expect(result.status).toBe('unknown') + expect(result.ownersByChatId.size).toBe(0) + }) +}) diff --git a/apps/sim/lib/copilot/request/session/abort.ts b/apps/sim/lib/copilot/request/session/abort.ts index ce508690361..b081044f8eb 100644 --- a/apps/sim/lib/copilot/request/session/abort.ts +++ b/apps/sim/lib/copilot/request/session/abort.ts @@ -35,6 +35,11 @@ const CHAT_STREAM_LOCK_TTL_SECONDS = 60 */ const CHAT_STREAM_LOCK_HEARTBEAT_INTERVAL_MS = 20_000 +export interface ChatStreamLockOwnersResult { + status: 'verified' | 'unknown' + ownersByChatId: Map +} + function registerPendingChatStream(chatId: string, streamId: string): void { let resolve!: () => void const promise = new Promise((r) => { @@ -123,6 +128,50 @@ export async function getPendingChatStreamId(chatId: string): Promise { + const localOwnersByChatId = new Map() + if (chatIds.length === 0) { + return { status: 'verified', ownersByChatId: localOwnersByChatId } + } + + for (const chatId of chatIds) { + const entry = pendingChatStreams.get(chatId) + if (entry?.streamId) localOwnersByChatId.set(chatId, entry.streamId) + } + + const redis = getRedisClient() + if (!redis) { + return { status: 'unknown', ownersByChatId: localOwnersByChatId } + } + + try { + const keys = chatIds.map(getChatStreamLockKey) + const values = await redis.mget(keys) + const redisOwnersByChatId = new Map() + for (let i = 0; i < chatIds.length; i++) { + const owner = values[i] + if (owner) redisOwnersByChatId.set(chatIds[i], owner) + } + return { status: 'verified', ownersByChatId: redisOwnersByChatId } + } catch (error) { + logger.warn('Failed to load chat stream lock owners (batch)', { + count: chatIds.length, + error: toError(error).message, + }) + return { status: 'unknown', ownersByChatId: localOwnersByChatId } + } +} + export async function releasePendingChatStream(chatId: string, streamId: string): Promise { try { await releaseLock(getChatStreamLockKey(chatId), streamId) diff --git a/apps/sim/lib/copilot/request/session/index.ts b/apps/sim/lib/copilot/request/session/index.ts index a09a194c788..c0ecb7a1716 100644 --- a/apps/sim/lib/copilot/request/session/index.ts +++ b/apps/sim/lib/copilot/request/session/index.ts @@ -1,9 +1,11 @@ +export type { ChatStreamLockOwnersResult } from './abort' export { AbortReason, type AbortReasonValue, abortActiveStream, acquirePendingChatStream, cleanupAbortMarker, + getChatStreamLockOwners, getPendingChatStreamId, isExplicitStopReason, registerActiveStream, diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 90c65eca62e..e16bda7c6ea 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -251,6 +251,7 @@ export interface SecureFetchResponse { status: number statusText: string headers: SecureFetchHeaders + body: ReadableStream | null text: () => Promise json: () => Promise arrayBuffer: () => Promise @@ -361,67 +362,89 @@ export async function secureFetchWithPinnedIP( return } - const chunks: Buffer[] = [] - let totalBytes = 0 - let responseTerminated = false - - res.on('data', (chunk: Buffer) => { - if (responseTerminated) return - - totalBytes += chunk.length - if ( - typeof maxResponseBytes === 'number' && - maxResponseBytes > 0 && - totalBytes > maxResponseBytes - ) { - responseTerminated = true - res.destroy(new Error(`Response exceeded maximum size of ${maxResponseBytes} bytes`)) - return + const headersRecord: Record = {} + let setCookieArray: string[] = [] + for (const [key, value] of Object.entries(res.headers)) { + const lowerKey = key.toLowerCase() + if (lowerKey === 'set-cookie') { + if (Array.isArray(value)) { + setCookieArray = value + headersRecord[lowerKey] = value.join(', ') + } else if (typeof value === 'string') { + setCookieArray = [value] + headersRecord[lowerKey] = value + } + } else if (typeof value === 'string') { + headersRecord[lowerKey] = value + } else if (Array.isArray(value)) { + headersRecord[lowerKey] = value.join(', ') } + } - chunks.push(chunk) - }) - - res.on('error', (error) => { - settledReject(error) + let totalBytes = 0 + const nodeRes = res + const body = new ReadableStream({ + start(controller) { + nodeRes.on('data', (chunk: Buffer) => { + totalBytes += chunk.length + if ( + typeof maxResponseBytes === 'number' && + maxResponseBytes > 0 && + totalBytes > maxResponseBytes + ) { + cleanupAbort() + controller.error( + new Error(`Response exceeded maximum size of ${maxResponseBytes} bytes`) + ) + nodeRes.destroy() + return + } + controller.enqueue(new Uint8Array(chunk)) + }) + nodeRes.on('end', () => { + cleanupAbort() + controller.close() + }) + nodeRes.on('error', (err) => { + cleanupAbort() + controller.error(err) + }) + }, + cancel() { + cleanupAbort() + nodeRes.destroy() + }, }) - res.on('end', () => { - if (responseTerminated) return - const bodyBuffer = Buffer.concat(chunks) - const body = bodyBuffer.toString('utf-8') - const headersRecord: Record = {} - let setCookieArray: string[] = [] - for (const [key, value] of Object.entries(res.headers)) { - const lowerKey = key.toLowerCase() - if (lowerKey === 'set-cookie') { - if (Array.isArray(value)) { - setCookieArray = value - headersRecord[lowerKey] = value.join(', ') - } else if (typeof value === 'string') { - setCookieArray = [value] - headersRecord[lowerKey] = value + let bodyBufferPromise: Promise | null = null + function readBodyAsBuffer(): Promise { + if (!bodyBufferPromise) { + bodyBufferPromise = (async () => { + const reader = body.getReader() + const buffers: Uint8Array[] = [] + while (true) { + const { done, value } = await reader.read() + if (done) break + if (value) buffers.push(value) } - } else if (typeof value === 'string') { - headersRecord[lowerKey] = value - } else if (Array.isArray(value)) { - headersRecord[lowerKey] = value.join(', ') - } + return Buffer.concat(buffers.map((b) => Buffer.from(b))) + })() } + return bodyBufferPromise + } - settledResolve({ - ok: statusCode >= 200 && statusCode < 300, - status: statusCode, - statusText: res.statusMessage || '', - headers: new SecureFetchHeaders(headersRecord, setCookieArray), - text: async () => body, - json: async () => JSON.parse(body), - arrayBuffer: async () => - bodyBuffer.buffer.slice( - bodyBuffer.byteOffset, - bodyBuffer.byteOffset + bodyBuffer.byteLength - ), - }) + settledResolve({ + ok: statusCode >= 200 && statusCode < 300, + status: statusCode, + statusText: res.statusMessage || '', + headers: new SecureFetchHeaders(headersRecord, setCookieArray), + body, + text: async () => (await readBodyAsBuffer()).toString('utf-8'), + json: async () => JSON.parse((await readBodyAsBuffer()).toString('utf-8')), + arrayBuffer: async () => { + const buf = await readBodyAsBuffer() + return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) as ArrayBuffer + }, }) }) @@ -433,7 +456,6 @@ export async function secureFetchWithPinnedIP( } } const settledResolve: typeof resolve = (value) => { - cleanupAbort() resolve(value) } const settledReject: typeof reject = (reason) => { diff --git a/apps/sim/lib/core/security/input-validation.ts b/apps/sim/lib/core/security/input-validation.ts index 12591aeb25a..98ac9e1c982 100644 --- a/apps/sim/lib/core/security/input-validation.ts +++ b/apps/sim/lib/core/security/input-validation.ts @@ -1593,3 +1593,30 @@ export function validateWorkdayTenantUrl( return { isValid: true, sanitized: url as string } } + +/** + * Validates a database identifier (table or column name) to prevent SQL injection. + * + * Accepts only identifiers that start with a letter or underscore and contain + * only letters, digits, and underscores — the safe subset of SQL identifiers. + * + * @param value - The identifier to validate + * @param paramName - Name of the parameter for error messages (e.g. 'table', 'column') + * @returns ValidationResult with isValid flag and optional error message + */ +export function validateDatabaseIdentifier( + value: unknown, + paramName = 'identifier' +): ValidationResult { + if (typeof value !== 'string' || value.length === 0) { + return { isValid: false, error: `${paramName} is required` } + } + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(value)) { + logger.warn('Invalid database identifier', { paramName, value: value.substring(0, 100) }) + return { + isValid: false, + error: `Invalid ${paramName}: must start with a letter or underscore and contain only letters, digits, and underscores`, + } + } + return { isValid: true, sanitized: value } +} diff --git a/apps/sim/lib/core/utils/response-format.ts b/apps/sim/lib/core/utils/response-format.ts index 7223f17e0fa..7512cc50b59 100644 --- a/apps/sim/lib/core/utils/response-format.ts +++ b/apps/sim/lib/core/utils/response-format.ts @@ -1,4 +1,6 @@ import { createLogger } from '@sim/logger' +import { materializeLargeValueRefSyncOrThrow } from '@/lib/execution/payloads/cache' +import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref' const logger = createLogger('ResponseFormatUtils') @@ -196,6 +198,10 @@ function traverseObjectPathInternal(obj: any, path: string): any { const parts = path.split('.') for (const part of parts) { + if (isLargeValueRef(current)) { + current = materializeLargeValueRefSyncOrThrow(current) + } + if (current?.[part] !== undefined) { current = current[part] } else { @@ -203,6 +209,10 @@ function traverseObjectPathInternal(obj: any, path: string): any { } } + if (isLargeValueRef(current)) { + return current + } + return current } diff --git a/apps/sim/lib/core/utils/user-file.ts b/apps/sim/lib/core/utils/user-file.ts index 0069eb4fbae..deee12cbf04 100644 --- a/apps/sim/lib/core/utils/user-file.ts +++ b/apps/sim/lib/core/utils/user-file.ts @@ -42,6 +42,27 @@ export function isUserFileWithMetadata(value: unknown): value is UserFile { return typeof candidate.size === 'number' && typeof candidate.type === 'string' } +/** + * Checks if a value matches the display-safe UserFile metadata shape after internal fields are stripped. + */ +export function isUserFileDisplayMetadata(value: unknown): value is Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return false + } + + const candidate = value as Record + const url = typeof candidate.url === 'string' ? candidate.url : '' + + return ( + typeof candidate.id === 'string' && + typeof candidate.name === 'string' && + url.length > 0 && + typeof candidate.size === 'number' && + typeof candidate.type === 'string' && + (candidate.id.startsWith('file_') || url.includes('/api/files/serve/')) + ) +} + /** * Filters a UserFile object to only include display fields. * Used for both UI display and log sanitization. diff --git a/apps/sim/lib/execution/event-buffer.test.ts b/apps/sim/lib/execution/event-buffer.test.ts index 7e03ab8954d..cd1753570ec 100644 --- a/apps/sim/lib/execution/event-buffer.test.ts +++ b/apps/sim/lib/execution/event-buffer.test.ts @@ -8,6 +8,7 @@ import type { ExecutionEvent } from '@/lib/workflows/executor/execution-events' const { mockGetRedisClient, mockRedis, persistedEntries } = vi.hoisted(() => { const persistedEntries: ExecutionEventEntry[] = [] const mockRedis = { + get: vi.fn(), incrby: vi.fn(), hset: vi.fn(), expire: vi.fn(), @@ -30,6 +31,7 @@ import { flushExecutionStreamReplayBuffer, initializeExecutionStreamMeta, readExecutionEventsState, + resetExecutionStreamBuffer, } from '@/lib/execution/event-buffer' function makeEvent(blockId: string): ExecutionEvent { @@ -47,36 +49,53 @@ function makeEvent(blockId: string): ExecutionEvent { } } +function parseFlushEvalArgs(args: unknown[]): { + terminalStatus: string + zaddArgs: (string | number)[] +} { + const keyCount = Number(args[0]) + return { + terminalStatus: String(args[keyCount + 4] ?? ''), + zaddArgs: args.slice(keyCount + 9) as (string | number)[], + } +} + +function isFlushScript(script: string): boolean { + return script.includes("redis.call('ZADD'") && script.includes('new_count') +} + +function isResetScript(script: string): boolean { + return script.includes('retained_bytes') && script.includes('replayStartEventId') +} + describe('execution event buffer', () => { beforeEach(() => { vi.clearAllMocks() persistedEntries.length = 0 mockGetRedisClient.mockReturnValue(mockRedis) + mockRedis.get.mockResolvedValue(null) mockRedis.hgetall.mockResolvedValue({}) mockRedis.zrangebyscore.mockResolvedValue([]) mockRedis.zremrangebyrank.mockResolvedValue(0) - mockRedis.eval.mockImplementation( - async ( - _script: string, - _keyCount: number, - _eventsKey: string, - _seqKey: string, - _metaKey: string, - _ttl: number, - _eventLimit: number, - _updatedAt: string, - terminalStatus: string, - ...args: (string | number)[] - ) => { - for (let i = 0; i < args.length; i += 2) { - persistedEntries.push(JSON.parse(args[i + 1] as string) as ExecutionEventEntry) + mockRedis.eval.mockImplementation(async (script: string, ...args: unknown[]) => { + if (isFlushScript(script)) { + const { terminalStatus, zaddArgs } = parseFlushEvalArgs(args) + for (let i = 0; i < zaddArgs.length; i += 2) { + persistedEntries.push(JSON.parse(zaddArgs[i + 1] as string) as ExecutionEventEntry) } if (terminalStatus) { await mockRedis.hset('meta', { status: terminalStatus }) } - return persistedEntries[0]?.eventId ?? false + return [1, persistedEntries[0]?.eventId ?? false, 0] } - ) + if (isResetScript(script)) { + return 0 + } + if (script.includes('DECRBY')) { + return 1 + } + return [1, 'ok', 0, 0] + }) mockRedis.pipeline.mockImplementation(() => ({ zadd: vi.fn((_key: string, ...args: (string | number)[]) => { for (let i = 0; i < args.length; i += 2) { @@ -152,15 +171,15 @@ describe('execution event buffer', () => { () => Promise.resolve(), ] - mockRedis.eval.mockImplementation(async (_script: string, ...args: unknown[]) => { + mockRedis.eval.mockImplementation(async (script: string, ...args: unknown[]) => { const batchEntries: ExecutionEventEntry[] = [] - const zaddArgs = args.slice(8) as (string | number)[] + const { zaddArgs } = parseFlushEvalArgs(args) for (let i = 0; i < zaddArgs.length; i += 2) { batchEntries.push(JSON.parse(zaddArgs[i + 1] as string) as ExecutionEventEntry) } await (execCalls.shift() ?? (() => Promise.resolve()))() persistedEntries.push(...batchEntries) - return persistedEntries[0]?.eventId ?? false + return [1, persistedEntries[0]?.eventId ?? false, 0] }) mockRedis.pipeline.mockImplementation(() => { const batchEntries: ExecutionEventEntry[] = [] @@ -237,8 +256,8 @@ describe('execution event buffer', () => { it('flushes replay events after a recovered final replay flush without terminal meta', async () => { mockRedis.incrby.mockResolvedValue(100) let flushAttempt = 0 - mockRedis.eval.mockImplementation(async (_script: string, ...args: unknown[]) => { - const zaddArgs = args.slice(8) as (string | number)[] + mockRedis.eval.mockImplementation(async (script: string, ...args: unknown[]) => { + const { zaddArgs } = parseFlushEvalArgs(args) if (flushAttempt > 0) { for (let i = 0; i < zaddArgs.length; i += 2) { persistedEntries.push(JSON.parse(zaddArgs[i + 1] as string) as ExecutionEventEntry) @@ -247,7 +266,7 @@ describe('execution event buffer', () => { if (flushAttempt++ === 0) { throw new Error('first flush failed') } - return persistedEntries[0]?.eventId ?? false + return [1, persistedEntries[0]?.eventId ?? false, 0] }) mockRedis.pipeline.mockImplementation(() => ({ zadd: vi.fn((_key: string, ...args: (string | number)[]) => { @@ -287,6 +306,99 @@ describe('execution event buffer', () => { expect(mockRedis.hset).toHaveBeenCalledWith('meta', { status: 'complete' }) }) + it('budgets only net event bytes after pruning during flush', async () => { + mockRedis.incrby.mockResolvedValue(100) + let netBudgetBytes = 0 + mockRedis.eval.mockImplementation(async (script: string, ...args: unknown[]) => { + const keyCount = Number(args[0]) + netBudgetBytes = Number(args[keyCount + 5]) + const { zaddArgs } = parseFlushEvalArgs(args) + for (let i = 0; i < zaddArgs.length; i += 2) { + persistedEntries.push(JSON.parse(zaddArgs[i + 1] as string) as ExecutionEventEntry) + } + return [1, persistedEntries[0]?.eventId ?? false, 123] + }) + + const writer = createExecutionEventWriter('exec-1') + await writer.writeTerminal(makeEvent('terminal'), 'complete') + + expect(netBudgetBytes).toBeGreaterThan(0) + }) + + it('releases retained event budget when resetting the stream buffer', async () => { + mockRedis.get.mockResolvedValueOnce(41) + mockRedis.hgetall.mockResolvedValueOnce({ userId: 'user-1' }) + let releasedBytes = 0 + mockRedis.eval.mockImplementationOnce(async (script: string, ...args: unknown[]) => { + expect(script).toContain('retained_bytes') + expect(args.slice(0, 5)).toEqual([ + 4, + 'execution:stream:exec-1:events', + 'execution:stream:exec-1:meta', + 'execution:redis-budget:execution:exec-1', + 'execution:redis-budget:user:user-1', + ]) + releasedBytes = 256 + return releasedBytes + }) + + await expect(resetExecutionStreamBuffer('exec-1')).resolves.toBe(true) + + expect(releasedBytes).toBe(256) + }) + + it('surfaces execution memory limit errors when the Redis budget is exceeded', async () => { + mockRedis.incrby.mockResolvedValue(100) + mockRedis.eval.mockImplementation(async (script: string) => { + if (isFlushScript(script)) { + return [0, 'execution_redis_bytes', 64 * 1024 * 1024] + } + return [1, 'ok', 0, 0] + }) + + const writer = createExecutionEventWriter('exec-1') + + await expect(writer.writeTerminal(makeEvent('terminal'), 'complete')).rejects.toThrow( + 'Execution memory limit exceeded' + ) + expect(persistedEntries).toEqual([]) + }) + + it('preserves requested UserFile base64 when buffering terminal events', async () => { + mockRedis.incrby.mockResolvedValue(100) + const base64 = Buffer.from('hello').toString('base64') + const writer = createExecutionEventWriter('exec-1', { preserveUserFileBase64: true }) + + await writer.writeTerminal( + { + type: 'execution:completed', + timestamp: new Date().toISOString(), + executionId: 'exec-1', + workflowId: 'wf-1', + data: { + success: true, + duration: 1, + output: { + file: { + id: 'file-1', + name: 'small.txt', + size: 5, + type: 'text/plain', + context: 'execution', + base64, + }, + }, + }, + }, + 'complete' + ) + + const eventData = persistedEntries[0].event.data as { + output: { file: { base64?: string } } + } + expect(eventData.output.file.base64).toBe(base64) + }) + it('retries active meta initialization before giving up', async () => { mockRedis.hset.mockRejectedValueOnce(new Error('meta write failed')).mockResolvedValueOnce(1) diff --git a/apps/sim/lib/execution/event-buffer.ts b/apps/sim/lib/execution/event-buffer.ts index 02f5d750b18..8c0d08090cb 100644 --- a/apps/sim/lib/execution/event-buffer.ts +++ b/apps/sim/lib/execution/event-buffer.ts @@ -2,6 +2,18 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { env } from '@/lib/core/config/env' import { getRedisClient } from '@/lib/core/config/redis' +import { LARGE_VALUE_THRESHOLD_BYTES } from '@/lib/execution/payloads/large-value-ref' +import { compactExecutionPayload } from '@/lib/execution/payloads/serializer' +import type { LargeValueStoreContext } from '@/lib/execution/payloads/store' +import { + type ExecutionRedisBudgetReservation, + getExecutionRedisBudgetKeys, + getExecutionRedisBudgetLimits, +} from '@/lib/execution/redis-budget.server' +import { + ExecutionResourceLimitError, + isExecutionResourceLimitError, +} from '@/lib/execution/resource-errors' import type { ExecutionEvent } from '@/lib/workflows/executor/execution-events' const logger = createLogger('ExecutionEventBuffer') @@ -11,13 +23,94 @@ const TTL_SECONDS = 60 * 60 // 1 hour const EVENT_LIMIT = 1000 const RESERVE_BATCH = 100 const FLUSH_INTERVAL_MS = 15 +const FLUSH_MAX_RETRY_INTERVAL_MS = 1000 const FLUSH_MAX_BATCH = 200 const MAX_PENDING_EVENTS = 1000 const ACTIVE_META_ATTEMPTS = 3 const FINALIZE_FLUSH_ATTEMPTS = 2 const FLUSH_EVENTS_SCRIPT = ` local terminal_status = ARGV[4] -for i = 5, #ARGV, 2 do +local batch_bytes = tonumber(ARGV[5]) +local execution_limit = tonumber(ARGV[6]) +local user_limit = tonumber(ARGV[7]) +local budget_ttl_seconds = tonumber(ARGV[8]) +local event_limit = tonumber(ARGV[2]) +local new_count = 0 +local new_bytes = 0 +local new_entries = {} +for i = 9, #ARGV, 2 do + local entry = ARGV[i + 1] + if not redis.call('ZSCORE', KEYS[1], entry) then + new_count = new_count + 1 + new_bytes = new_bytes + string.len(entry) + table.insert(new_entries, entry) + end +end +local current_count = redis.call('ZCARD', KEYS[1]) +local prune_count = current_count + new_count - event_limit +local pruned = {} +if prune_count < 0 then + prune_count = 0 +end +local existing_prune_count = math.min(prune_count, current_count) +local new_prune_count = prune_count - existing_prune_count +if existing_prune_count > 0 then + pruned = redis.call('ZRANGE', KEYS[1], 0, existing_prune_count - 1) +end +local pruned_bytes = 0 +for _, entry in ipairs(pruned) do + pruned_bytes = pruned_bytes + string.len(entry) +end +for i = 1, new_prune_count do + local entry = new_entries[i] + if entry then + pruned_bytes = pruned_bytes + string.len(entry) + end +end +local net_bytes = new_bytes - pruned_bytes +if net_bytes > 0 then + local execution_current = tonumber(redis.call('GET', KEYS[4]) or '0') + if execution_limit > 0 and execution_current + net_bytes > execution_limit then + return {0, 'execution_redis_bytes', execution_current, pruned_bytes} + end + local user_current = 0 + if #KEYS >= 5 then + user_current = tonumber(redis.call('GET', KEYS[5]) or '0') + if user_limit > 0 and user_current + net_bytes > user_limit then + return {0, 'user_redis_bytes', user_current, pruned_bytes} + end + end + redis.call('INCRBY', KEYS[4], net_bytes) + redis.call('EXPIRE', KEYS[4], budget_ttl_seconds) + if #KEYS >= 5 then + redis.call('INCRBY', KEYS[5], net_bytes) + redis.call('EXPIRE', KEYS[5], budget_ttl_seconds) + end +elseif net_bytes < 0 then + local release_bytes = -net_bytes + local execution_next = redis.call('DECRBY', KEYS[4], release_bytes) + if execution_next <= 0 then + redis.call('DEL', KEYS[4]) + else + redis.call('EXPIRE', KEYS[4], budget_ttl_seconds) + end + if #KEYS >= 5 then + local user_next = redis.call('DECRBY', KEYS[5], release_bytes) + if user_next <= 0 then + redis.call('DEL', KEYS[5]) + else + redis.call('EXPIRE', KEYS[5], budget_ttl_seconds) + end + end +else + if redis.call('EXISTS', KEYS[4]) == 1 then + redis.call('EXPIRE', KEYS[4], budget_ttl_seconds) + end + if #KEYS >= 5 and redis.call('EXISTS', KEYS[5]) == 1 then + redis.call('EXPIRE', KEYS[5], budget_ttl_seconds) + end +end +for i = 9, #ARGV, 2 do redis.call('ZADD', KEYS[1], ARGV[i], ARGV[i + 1]) end redis.call('EXPIRE', KEYS[1], tonumber(ARGV[1])) @@ -32,7 +125,34 @@ if oldest[2] then redis.call('HSET', KEYS[3], 'earliestEventId', tostring(math.floor(tonumber(oldest[2]))), 'updatedAt', ARGV[3]) redis.call('EXPIRE', KEYS[3], tonumber(ARGV[1])) end -return oldest[2] or false +return {1, oldest[2] or false, pruned_bytes} +` +const RESET_STREAM_SCRIPT = ` +local entries = redis.call('ZRANGE', KEYS[1], 0, -1) +local retained_bytes = 0 +for _, entry in ipairs(entries) do + retained_bytes = retained_bytes + string.len(entry) +end +redis.call('DEL', KEYS[1], KEYS[2]) +redis.call('HSET', KEYS[2], 'replayStartEventId', ARGV[1], 'updatedAt', ARGV[2]) +redis.call('EXPIRE', KEYS[2], tonumber(ARGV[3])) +if retained_bytes > 0 then + local execution_next = redis.call('DECRBY', KEYS[3], retained_bytes) + if execution_next <= 0 then + redis.call('DEL', KEYS[3]) + else + redis.call('EXPIRE', KEYS[3], tonumber(ARGV[4])) + end + if #KEYS >= 4 then + local user_next = redis.call('DECRBY', KEYS[4], retained_bytes) + if user_next <= 0 then + redis.call('DEL', KEYS[4]) + else + redis.call('EXPIRE', KEYS[4], tonumber(ARGV[4])) + end + end +end +return retained_bytes ` function getEventsKey(executionId: string) { @@ -53,6 +173,69 @@ function isExecutionStreamStatus(value: string | undefined): value is ExecutionS return value === 'active' || value === 'complete' || value === 'error' || value === 'cancelled' } +function getJsonSize(value: unknown): number | null { + try { + return Buffer.byteLength(JSON.stringify(value), 'utf8') + } catch { + return null + } +} + +function getExecutionEventEntryJson(entry: ExecutionEventEntry): string { + return JSON.stringify(entry) +} + +function getFlushScriptResult(value: unknown): { + allowed: boolean + resource?: string + currentBytes?: number +} { + if (Array.isArray(value)) { + return { + allowed: Number(value[0]) === 1, + resource: typeof value[1] === 'string' ? value[1] : undefined, + currentBytes: Number(value[2] ?? 0), + } + } + return { allowed: true } +} + +function trimFinalBlockLogsForEventData(data: unknown): unknown { + if (!data || typeof data !== 'object' || Array.isArray(data)) return data + + const record = data as Record + const finalBlockLogs = record.finalBlockLogs + if (!Array.isArray(finalBlockLogs)) return data + const originalSize = getJsonSize(data) + if (originalSize !== null && originalSize <= LARGE_VALUE_THRESHOLD_BYTES) return data + + const total = finalBlockLogs.length + let logs = finalBlockLogs + let trimmed: Record = { + ...record, + finalBlockLogs: logs, + finalBlockLogsTruncated: true, + finalBlockLogsTotal: total, + } + + while (logs.length > 0) { + const size = getJsonSize(trimmed) + if (size !== null && size <= LARGE_VALUE_THRESHOLD_BYTES) { + return trimmed + } + + logs = logs.length === 1 ? [] : logs.slice(Math.ceil(logs.length / 2)) + trimmed = { + ...record, + finalBlockLogs: logs, + finalBlockLogsTruncated: true, + finalBlockLogsTotal: total, + } + } + + return trimmed +} + export interface ExecutionStreamMeta { status: ExecutionStreamStatus userId?: string @@ -97,6 +280,70 @@ export interface ExecutionEventWriter { close: () => Promise } +export interface ExecutionEventWriterContext extends LargeValueStoreContext { + requireDurablePayloads?: boolean + preserveUserFileBase64?: boolean +} + +async function compactEventForBuffer( + event: ExecutionEvent, + context: ExecutionEventWriterContext = {} +): Promise { + if (!('data' in event)) { + return event + } + + const baseOptions = { + ...context, + executionId: context.executionId ?? event.executionId, + requireDurable: context.requireDurablePayloads, + preserveRoot: true, + } + + let compactedData = await compactExecutionPayload(event.data, { + ...baseOptions, + preserveUserFileBase64: context.preserveUserFileBase64, + }) + let eventData = trimFinalBlockLogsForEventData(compactedData) + let eventDataSize = getJsonSize(eventData) + + // SSE/replay events are size-bounded by LARGE_VALUE_THRESHOLD_BYTES. When a + // payload that preserved UserFile base64 (e.g., for chat/streaming) exceeds + // the cap, recompact the already-compacted result with base64 stripped so + // consumers can lazily re-hydrate via sim.files.readBase64. Recompacting the + // *compacted* value (not the raw event.data) lets existing LargeValueRefs + // pass through unchanged and avoids minting fresh storage objects for the + // same large fields. + if ( + context.preserveUserFileBase64 && + eventDataSize !== null && + eventDataSize > LARGE_VALUE_THRESHOLD_BYTES + ) { + const oversizedBytes = eventDataSize + compactedData = await compactExecutionPayload(compactedData, { + ...baseOptions, + preserveUserFileBase64: false, + }) + eventData = trimFinalBlockLogsForEventData(compactedData) + eventDataSize = getJsonSize(eventData) + logger.warn('Stripped inline UserFile base64 from execution event to fit size limit', { + executionId: baseOptions.executionId, + eventType: 'type' in event ? event.type : undefined, + thresholdBytes: LARGE_VALUE_THRESHOLD_BYTES, + originalBytes: oversizedBytes, + strippedBytes: eventDataSize, + }) + } + + if (eventDataSize !== null && eventDataSize > LARGE_VALUE_THRESHOLD_BYTES) { + throw new Error( + `Execution event data remains too large after compaction (${eventDataSize} bytes)` + ) + } + + return { ...event, data: eventData } as ExecutionEvent +} + const memoryExecutionStreams = new Map() function canUseMemoryEventBuffer(): boolean { @@ -169,13 +416,17 @@ function readMemoryEvents(executionId: string, afterEventId: number): ExecutionE } } -function createMemoryExecutionEventWriter(executionId: string): ExecutionEventWriter { +function createMemoryExecutionEventWriter( + executionId: string, + context: ExecutionEventWriterContext = {} +): ExecutionEventWriter { const writeMemoryEvent = async (event: ExecutionEvent) => { const stream = getMemoryStream(executionId) + const compactEvent = await compactEventForBuffer(event, context) const entry = { eventId: stream.nextEventId++, executionId, - event, + event: compactEvent, } stream.events.push(entry) if (stream.events.length > EVENT_LIMIT) { @@ -235,7 +486,12 @@ export async function flushExecutionStreamReplayBuffer( } export async function resetExecutionStreamBuffer(executionId: string): Promise { - if (canUseMemoryEventBuffer()) { + const redis = getRedisClient() + if (!redis) { + if (!canUseMemoryEventBuffer()) { + logger.warn('resetExecutionStreamBuffer: Redis client unavailable', { executionId }) + return false + } const stream = getMemoryStream(executionId) stream.events = [] stream.meta = { @@ -247,22 +503,32 @@ export async function resetExecutionStreamBuffer(executionId: string): Promise 0)) const replayStartEventId = Number.isFinite(currentSequence) ? currentSequence + 1 : 1 const metaKey = getMetaKey(executionId) - await redis.del(getEventsKey(executionId), metaKey) - await redis.hset(metaKey, { - replayStartEventId: String(replayStartEventId), - updatedAt: new Date().toISOString(), - }) - await redis.expire(metaKey, TTL_SECONDS) + const meta = (await redis.hgetall(metaKey).catch(() => ({}))) as Record + const userId = typeof meta.userId === 'string' ? meta.userId : undefined + const budgetReservation: ExecutionRedisBudgetReservation = { + executionId, + userId, + category: 'event_buffer', + operation: 'reset_events', + bytes: 0, + logger, + } + const budgetKeys = getExecutionRedisBudgetKeys(budgetReservation) + await redis.eval( + RESET_STREAM_SCRIPT, + 2 + budgetKeys.length, + getEventsKey(executionId), + metaKey, + ...budgetKeys, + String(replayStartEventId), + new Date().toISOString(), + TTL_SECONDS, + getExecutionRedisBudgetLimits().ttlSeconds + ) return true } catch (error) { logger.warn('Failed to reset execution stream buffer', { @@ -450,12 +716,15 @@ export async function readExecutionEventsState( } } -export function createExecutionEventWriter(executionId: string): ExecutionEventWriter { +export function createExecutionEventWriter( + executionId: string, + context: ExecutionEventWriterContext = {} +): ExecutionEventWriter { const redis = getRedisClient() if (!redis) { if (canUseMemoryEventBuffer()) { logger.info('createExecutionEventWriter: using in-memory event buffer', { executionId }) - return createMemoryExecutionEventWriter(executionId) + return createMemoryExecutionEventWriter(executionId, context) } logger.warn( 'createExecutionEventWriter: Redis client unavailable, events will not be buffered', @@ -477,13 +746,23 @@ export function createExecutionEventWriter(executionId: string): ExecutionEventW let nextEventId = 0 let maxReservedId = 0 let flushTimer: ReturnType | null = null + let consecutiveFlushFailures = 0 - const scheduleFlush = () => { + const getFlushDelayMs = () => { + if (consecutiveFlushFailures === 0) return FLUSH_INTERVAL_MS + const backoff = Math.min( + FLUSH_INTERVAL_MS * 2 ** Math.min(consecutiveFlushFailures, 6), + FLUSH_MAX_RETRY_INTERVAL_MS + ) + return backoff + Math.floor(Math.random() * FLUSH_INTERVAL_MS) + } + + const scheduleFlush = (delayMs = FLUSH_INTERVAL_MS) => { if (flushTimer) return flushTimer = setTimeout(() => { flushTimer = null void flushPending() - }, FLUSH_INTERVAL_MS) + }, delayMs) } const reserveIds = async (minCount: number) => { @@ -509,26 +788,74 @@ export function createExecutionEventWriter(executionId: string): ExecutionEventW try { const key = getEventsKey(executionId) const zaddArgs: (string | number)[] = [] + let batchBytes = 0 for (const entry of batch) { - zaddArgs.push(entry.eventId, JSON.stringify(entry)) + const entryJson = getExecutionEventEntryJson(entry) + batchBytes += Buffer.byteLength(entryJson, 'utf8') + zaddArgs.push(entry.eventId, entryJson) + } + const budgetReservation: ExecutionRedisBudgetReservation = { + executionId, + userId: context.userId, + category: 'event_buffer', + operation: terminalStatus ? 'write_terminal_events' : 'write_events', + bytes: batchBytes, + logger, } - await redis.eval( - FLUSH_EVENTS_SCRIPT, - 3, - key, - getSeqKey(executionId), - getMetaKey(executionId), - TTL_SECONDS, - EVENT_LIMIT, - new Date().toISOString(), - terminalStatus ?? '', - ...zaddArgs + const limits = getExecutionRedisBudgetLimits() + if (batchBytes > limits.maxSingleWriteBytes) { + throw new ExecutionResourceLimitError({ + resource: 'redis_key_bytes', + attemptedBytes: batchBytes, + limitBytes: limits.maxSingleWriteBytes, + }) + } + const budgetKeys = getExecutionRedisBudgetKeys(budgetReservation) + const flushResult = getFlushScriptResult( + await redis.eval( + FLUSH_EVENTS_SCRIPT, + 3 + budgetKeys.length, + key, + getSeqKey(executionId), + getMetaKey(executionId), + ...budgetKeys, + TTL_SECONDS, + EVENT_LIMIT, + new Date().toISOString(), + terminalStatus ?? '', + batchBytes, + limits.maxExecutionBytes, + limits.maxUserBytes, + limits.ttlSeconds, + ...zaddArgs + ) ) + if (!flushResult.allowed) { + throw new ExecutionResourceLimitError({ + resource: + flushResult.resource === 'user_redis_bytes' + ? 'user_redis_bytes' + : 'execution_redis_bytes', + attemptedBytes: batchBytes, + currentBytes: flushResult.currentBytes ?? 0, + limitBytes: + flushResult.resource === 'user_redis_bytes' + ? limits.maxUserBytes + : limits.maxExecutionBytes, + }) + } + consecutiveFlushFailures = 0 return true } catch (error) { + if (isExecutionResourceLimitError(error)) { + pending = batch.concat(pending) + throw error + } + consecutiveFlushFailures += 1 logger.warn('Failed to flush execution events', { executionId, batchSize: batch.length, + consecutiveFailures: consecutiveFlushFailures, error: toError(error).message, stack: error instanceof Error ? error.stack : undefined, }) @@ -566,7 +893,7 @@ export function createExecutionEventWriter(executionId: string): ExecutionEventW flushPromise = null } if (!ok) { - if (scheduleOnFailure && pending.length > 0) scheduleFlush() + if (scheduleOnFailure && pending.length > 0) scheduleFlush(getFlushDelayMs()) return false } } @@ -577,7 +904,12 @@ export function createExecutionEventWriter(executionId: string): ExecutionEventW await reserveIds(1) } const eventId = nextEventId++ - const entry: ExecutionEventEntry = { eventId, executionId, event } + const compactEvent = await compactEventForBuffer(event, { + ...context, + executionId, + requireDurablePayloads: true, + }) + const entry: ExecutionEventEntry = { eventId, executionId, event: compactEvent } pending.push(entry) if (pending.length >= FLUSH_MAX_BATCH) { await flushPending() @@ -618,7 +950,12 @@ export function createExecutionEventWriter(executionId: string): ExecutionEventW await reserveIds(1) } const eventId = nextEventId++ - const entry: ExecutionEventEntry = { eventId, executionId, event } + const compactEvent = await compactEventForBuffer(event, { + ...context, + executionId, + requireDurablePayloads: true, + }) + const entry: ExecutionEventEntry = { eventId, executionId, event: compactEvent } pending.push(entry) const ok = await flushPending(false, status) if (!ok) { diff --git a/apps/sim/lib/execution/isolated-vm-worker.cjs b/apps/sim/lib/execution/isolated-vm-worker.cjs index 18828eebc60..a924beb8dfe 100644 --- a/apps/sim/lib/execution/isolated-vm-worker.cjs +++ b/apps/sim/lib/execution/isolated-vm-worker.cjs @@ -27,6 +27,21 @@ const SANDBOX_BUNDLE_FILES = { const bundleSourceCache = new Map() const activeIsolates = new Map() +/** + * Sends an IPC request and reports only actual delivery failures. + * Node queues messages under backpressure, so the boolean return value is not + * a failure signal. + */ +function sendIpcRequest(message, onError) { + try { + process.send(message, (err) => { + if (err) onError(err) + }) + } catch (error) { + onError(error instanceof Error ? error : new Error(String(error))) + } +} + function getBundleSource(bundleName) { const cached = bundleSourceCache.get(bundleName) if (cached) return cached @@ -180,6 +195,7 @@ async function executeCode(request, executionId) { let logCallback = null let errorCallback = null let fetchCallback = null + let brokerCallback = null const externalCopies = [] try { @@ -232,17 +248,50 @@ async function executeCode(request, executionId) { } }, FETCH_TIMEOUT_MS) pendingFetches.set(fetchId, { resolve, timeout }) - if (process.send && process.connected) { - process.send({ type: 'fetch', fetchId, requestId, url, optionsJson }) - } else { + if (!process.send || !process.connected) { clearTimeout(timeout) pendingFetches.delete(fetchId) resolve(JSON.stringify({ error: 'Parent process disconnected' })) + return } + sendIpcRequest({ type: 'fetch', fetchId, requestId, url, optionsJson }, (err) => { + const pending = pendingFetches.get(fetchId) + if (!pending) return + clearTimeout(pending.timeout) + pendingFetches.delete(fetchId) + pending.resolve(JSON.stringify({ error: `Fetch IPC send failed: ${err.message}` })) + }) }) }) await jail.set('__fetchRef', fetchCallback) + brokerCallback = new ivm.Reference(async (brokerName, argsJson) => { + return new Promise((resolve) => { + const brokerId = ++brokerIdCounter + const timeout = setTimeout(() => { + if (pendingBrokerCalls.has(brokerId)) { + pendingBrokerCalls.delete(brokerId) + resolve(JSON.stringify({ error: `Broker "${brokerName}" timed out` })) + } + }, BROKER_TIMEOUT_MS) + pendingBrokerCalls.set(brokerId, { resolve, timeout, executionId }) + if (!process.send || !process.connected) { + clearTimeout(timeout) + pendingBrokerCalls.delete(brokerId) + resolve(JSON.stringify({ error: 'Parent process disconnected' })) + return + } + sendIpcRequest({ type: 'broker', brokerId, executionId, brokerName, argsJson }, (err) => { + const pending = pendingBrokerCalls.get(brokerId) + if (!pending) return + clearTimeout(pending.timeout) + pendingBrokerCalls.delete(brokerId) + pending.resolve(JSON.stringify({ error: `Broker IPC send failed: ${err.message}` })) + }) + }) + }) + await jail.set('__brokerRef', brokerCallback) + const bootstrap = ` // Set up console object const console = { @@ -299,10 +348,57 @@ async function executeCode(request, executionId) { }; } + const sim = (() => { + const broker = __brokerRef; + async function callSimBroker(name, args) { + let argsJson; + try { + argsJson = args === undefined ? undefined : JSON.stringify(args); + } catch { + throw new Error('sim helper arguments must be JSON-serializable'); + } + if (argsJson && argsJson.length > ${MAX_FETCH_OPTIONS_JSON_CHARS}) { + throw new Error('sim helper arguments exceed maximum payload size'); + } + const responseJson = await broker.apply(undefined, [name, argsJson], { result: { promise: true } }); + let response; + try { + response = JSON.parse(responseJson); + } catch { + throw new Error('Invalid sim helper response'); + } + if (typeof response.error === 'string') { + throw new Error(response.error || 'Sim helper call failed'); + } + return response.resultJson === undefined || response.resultJson === null + ? null + : JSON.parse(response.resultJson); + } + + return Object.freeze({ + files: Object.freeze({ + readBase64: (file, options) => callSimBroker('sim.files.readBase64', { file, options }), + readText: (file, options) => callSimBroker('sim.files.readText', { file, options }), + readBase64Chunk: (file, options) => callSimBroker('sim.files.readBase64Chunk', { file, options }), + readTextChunk: (file, options) => callSimBroker('sim.files.readTextChunk', { file, options }), + }), + values: Object.freeze({ + read: (ref, options) => callSimBroker('sim.values.read', { ref, options }), + }), + }); + })(); + Object.defineProperty(global, 'sim', { + value: sim, + writable: false, + configurable: false, + enumerable: true + }); + // Prevent access to dangerous globals with stronger protection const undefined_globals = [ 'Isolate', 'Context', 'Script', 'Module', 'Callback', 'Reference', - 'ExternalCopy', 'process', 'require', 'module', 'exports', '__dirname', '__filename' + 'ExternalCopy', 'process', 'require', 'module', 'exports', '__dirname', '__filename', + '__brokerRef', '__broker', '__callSimBroker' ]; for (const name of undefined_globals) { try { @@ -439,6 +535,7 @@ async function executeCode(request, executionId) { bootstrapScript, ...externalCopies, fetchCallback, + brokerCallback, errorCallback, logCallback, context, @@ -662,13 +759,19 @@ async function executeTask(request, executionId) { } }, BROKER_TIMEOUT_MS) pendingBrokerCalls.set(brokerId, { resolve, timeout, executionId }) - if (process.send && process.connected) { - process.send({ type: 'broker', brokerId, executionId, brokerName, argsJson }) - } else { + if (!process.send || !process.connected) { clearTimeout(timeout) pendingBrokerCalls.delete(brokerId) resolve(JSON.stringify({ error: 'Parent process disconnected' })) + return } + sendIpcRequest({ type: 'broker', brokerId, executionId, brokerName, argsJson }, (err) => { + const pending = pendingBrokerCalls.get(brokerId) + if (!pending) return + clearTimeout(pending.timeout) + pendingBrokerCalls.delete(brokerId) + pending.resolve(JSON.stringify({ error: `Broker IPC send failed: ${err.message}` })) + }) }) }) releaseables.push(brokerRef) diff --git a/apps/sim/lib/execution/payloads/cache.ts b/apps/sim/lib/execution/payloads/cache.ts new file mode 100644 index 00000000000..507a8dd4ccc --- /dev/null +++ b/apps/sim/lib/execution/payloads/cache.ts @@ -0,0 +1,169 @@ +import { + getLargeValueMaterializationError, + isLargeValueRef, + type LargeValueRef, +} from '@/lib/execution/payloads/large-value-ref' + +const FALLBACK_TTL_MS = 15 * 60 * 1000 +const MAX_IN_MEMORY_BYTES = 256 * 1024 * 1024 + +interface LargeValueCacheScope { + workspaceId?: string + workflowId?: string + executionId?: string + largeValueExecutionIds?: string[] + allowLargeValueWorkflowScope?: boolean +} + +const inMemoryValues = new Map< + string, + { + value: unknown + size: number + expiresAt: number + scope?: LargeValueCacheScope + recoverable: boolean + } +>() +let inMemoryBytes = 0 + +export function clearLargeValueCacheForTests(): void { + inMemoryValues.clear() + inMemoryBytes = 0 +} + +function cleanupExpiredValues(now = Date.now()): void { + for (const [id, entry] of inMemoryValues.entries()) { + if (entry.expiresAt <= now) { + inMemoryValues.delete(id) + inMemoryBytes -= entry.size + } + } +} + +export function cacheLargeValue( + id: string, + value: unknown, + size: number, + scope?: LargeValueCacheScope, + options: { recoverable?: boolean } = {} +): boolean { + if (size > MAX_IN_MEMORY_BYTES) { + return false + } + + cleanupExpiredValues() + + const existing = inMemoryValues.get(id) + if (existing) { + inMemoryValues.delete(id) + inMemoryBytes -= existing.size + } + + while (inMemoryBytes + size > MAX_IN_MEMORY_BYTES && inMemoryValues.size > 0) { + const oldestRecoverableId = Array.from(inMemoryValues.entries()).find( + ([, entry]) => entry.recoverable + )?.[0] + if (!oldestRecoverableId) break + const oldest = inMemoryValues.get(oldestRecoverableId) + inMemoryValues.delete(oldestRecoverableId) + inMemoryBytes -= oldest?.size ?? 0 + } + + if (inMemoryBytes + size > MAX_IN_MEMORY_BYTES) { + if (existing) { + inMemoryValues.set(id, existing) + inMemoryBytes += existing.size + } + return false + } + + inMemoryValues.set(id, { + value, + size, + scope, + recoverable: options.recoverable ?? false, + expiresAt: Date.now() + FALLBACK_TTL_MS, + }) + inMemoryBytes += size + return true +} + +function scopeMatchesRef( + ref: LargeValueRef, + cachedScope: LargeValueCacheScope | undefined, + callerScope?: LargeValueCacheScope +): boolean { + if (!cachedScope?.executionId) { + return false + } + if (ref.executionId && ref.executionId !== cachedScope.executionId) { + return false + } + if (!callerScope) { + return Boolean(ref.key) && (!ref.executionId || ref.executionId === cachedScope.executionId) + } + + const allowedExecutionIds = new Set([ + callerScope.executionId, + ...(callerScope.largeValueExecutionIds ?? []), + ]) + const workflowScopeAllowed = + callerScope.allowLargeValueWorkflowScope && + callerScope.workspaceId === cachedScope.workspaceId && + callerScope.workflowId === cachedScope.workflowId + + return allowedExecutionIds.has(cachedScope.executionId) || Boolean(workflowScopeAllowed) +} + +export function materializeLargeValueRefSync( + ref: LargeValueRef, + callerScope?: LargeValueCacheScope +): unknown { + cleanupExpiredValues() + const cached = inMemoryValues.get(ref.id) + if (!cached || !scopeMatchesRef(ref, cached.scope, callerScope)) { + return undefined + } + return cached.value +} + +export function materializeLargeValueRefSyncOrThrow( + ref: LargeValueRef, + callerScope?: LargeValueCacheScope +): unknown { + const materialized = materializeLargeValueRefSync(ref, callerScope) + if (materialized === undefined) { + throw getLargeValueMaterializationError(ref) + } + return materialized +} + +export function materializeLargeValueRefsSync( + value: unknown, + seen = new WeakSet() +): unknown { + if (isLargeValueRef(value)) { + return materializeLargeValueRefsSync(materializeLargeValueRefSyncOrThrow(value), seen) + } + + if (!value || typeof value !== 'object') { + return value + } + + if (seen.has(value)) { + return value + } + seen.add(value) + + if (Array.isArray(value)) { + return value.map((item) => materializeLargeValueRefsSync(item, seen)) + } + + return Object.fromEntries( + Object.entries(value).map(([key, entryValue]) => [ + key, + materializeLargeValueRefsSync(entryValue, seen), + ]) + ) +} diff --git a/apps/sim/lib/execution/payloads/hydration.ts b/apps/sim/lib/execution/payloads/hydration.ts new file mode 100644 index 00000000000..bfc825280ae --- /dev/null +++ b/apps/sim/lib/execution/payloads/hydration.ts @@ -0,0 +1,35 @@ +import { isLargeValueRef } from '@/lib/execution/payloads/large-value-ref' +import { + type LargeValueStoreContext, + materializeLargeValueRef, +} from '@/lib/execution/payloads/store' + +export async function warmLargeValueRefs( + value: unknown, + context: LargeValueStoreContext = {}, + seen = new WeakSet() +): Promise { + if (!value || typeof value !== 'object') { + return + } + + if (isLargeValueRef(value)) { + const materialized = await materializeLargeValueRef(value, context) + await warmLargeValueRefs(materialized, context, seen) + return + } + + if (seen.has(value)) { + return + } + seen.add(value) + + if (Array.isArray(value)) { + await Promise.all(value.map((item) => warmLargeValueRefs(item, context, seen))) + return + } + + await Promise.all( + Object.values(value).map((entryValue) => warmLargeValueRefs(entryValue, context, seen)) + ) +} diff --git a/apps/sim/lib/execution/payloads/large-value-ref.ts b/apps/sim/lib/execution/payloads/large-value-ref.ts new file mode 100644 index 00000000000..d770f6ed37d --- /dev/null +++ b/apps/sim/lib/execution/payloads/large-value-ref.ts @@ -0,0 +1,97 @@ +export const LARGE_VALUE_REF_MARKER = '__simLargeValueRef' + +export const LARGE_VALUE_THRESHOLD_BYTES = 8 * 1024 * 1024 +export const LARGE_VALUE_REF_VERSION = 1 + +export const LARGE_VALUE_KINDS = ['array', 'object', 'string', 'json'] as const + +export type LargeValueKind = (typeof LARGE_VALUE_KINDS)[number] + +export interface LargeValueRef { + [LARGE_VALUE_REF_MARKER]: true + version: typeof LARGE_VALUE_REF_VERSION + id: string + kind: LargeValueKind + size: number + key?: string + executionId?: string + preview?: unknown +} + +const LARGE_VALUE_ID_PATTERN = /^lv_[A-Za-z0-9_-]{12}$/ + +export function isLargeValueStorageKey(key: string, id: string, executionId?: string): boolean { + if (!key.startsWith('execution/')) return false + if (!key.endsWith(`/large-value-${id}.json`)) return false + if (executionId && !key.includes(`/${executionId}/`)) return false + return true +} + +export function isLargeValueRef(value: unknown): value is LargeValueRef { + if (!value || typeof value !== 'object') return false + + const candidate = value as Record + const id = candidate.id + const key = candidate.key + const executionId = candidate.executionId + + return ( + candidate[LARGE_VALUE_REF_MARKER] === true && + candidate.version === LARGE_VALUE_REF_VERSION && + typeof id === 'string' && + LARGE_VALUE_ID_PATTERN.test(id) && + typeof candidate.kind === 'string' && + (LARGE_VALUE_KINDS as readonly string[]).includes(candidate.kind) && + typeof candidate.size === 'number' && + Number.isFinite(candidate.size) && + candidate.size > 0 && + (executionId === undefined || typeof executionId === 'string') && + (key === undefined || + (typeof key === 'string' && + isLargeValueStorageKey(key, id, executionId as string | undefined))) + ) +} + +export function containsLargeValueRef( + value: unknown, + seen = new WeakSet() +): LargeValueRef | null { + if (!value || typeof value !== 'object') return null + if (isLargeValueRef(value)) return value + if (seen.has(value)) return null + + seen.add(value) + + if (Array.isArray(value)) { + for (const item of value) { + const ref = containsLargeValueRef(item, seen) + if (ref) return ref + } + return null + } + + for (const entryValue of Object.values(value)) { + const ref = containsLargeValueRef(entryValue, seen) + if (ref) return ref + } + + return null +} + +export function getLargeValueMaterializationError(ref: LargeValueRef): Error { + return new Error( + `This execution value is too large to inline (${formatLargeValueSize(ref.size)}). Select a nested field or reduce the amount of data passed between blocks.` + ) +} + +function formatLargeValueSize(bytes: number): string { + const megabytes = bytes / (1024 * 1024) + return `${megabytes.toFixed(1)} MB` +} + +export function assertNoLargeValueRefs(value: unknown): void { + const ref = containsLargeValueRef(value) + if (ref) { + throw getLargeValueMaterializationError(ref) + } +} diff --git a/apps/sim/lib/execution/payloads/materialization.server.ts b/apps/sim/lib/execution/payloads/materialization.server.ts new file mode 100644 index 00000000000..5e337e35914 --- /dev/null +++ b/apps/sim/lib/execution/payloads/materialization.server.ts @@ -0,0 +1,294 @@ +import { createLogger, type Logger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' +import { + getLargeValueMaterializationError, + isLargeValueRef, + isLargeValueStorageKey, + type LargeValueRef, +} from '@/lib/execution/payloads/large-value-ref' +import { ExecutionResourceLimitError } from '@/lib/execution/resource-errors' +import type { StorageContext } from '@/lib/uploads' +import { bufferToBase64, inferContextFromKey } from '@/lib/uploads/utils/file-utils' +import { downloadFileFromStorage } from '@/lib/uploads/utils/file-utils.server' +import type { UserFile } from '@/executor/types' + +const logger = createLogger('ExecutionPayloadMaterialization') + +export const MAX_DURABLE_LARGE_VALUE_BYTES = 64 * 1024 * 1024 +export const MAX_INLINE_MATERIALIZATION_BYTES = 16 * 1024 * 1024 +export const MAX_FUNCTION_FILE_BYTES = 64 * 1024 * 1024 +export const MAX_FUNCTION_INLINE_BYTES = 10 * 1024 * 1024 + +export interface ExecutionMaterializationContext { + workflowId?: string + workspaceId?: string + executionId?: string + largeValueExecutionIds?: string[] + allowLargeValueWorkflowScope?: boolean + userId?: string + requestId?: string + logger?: Logger +} + +export interface MaterializeLargeValueOptions extends ExecutionMaterializationContext { + maxBytes?: number +} + +export interface ReadUserFileContentOptions extends ExecutionMaterializationContext { + maxBytes?: number + maxSourceBytes?: number + offset?: number + length?: number + chunked?: boolean + encoding: 'base64' | 'text' +} + +function getLogger(options: ExecutionMaterializationContext): Logger { + return options.logger ?? logger +} + +export function assertDurableLargeValueSize(size: number): void { + if (size > MAX_DURABLE_LARGE_VALUE_BYTES) { + throw new ExecutionResourceLimitError({ + resource: 'execution_payload_bytes', + attemptedBytes: size, + limitBytes: MAX_DURABLE_LARGE_VALUE_BYTES, + }) + } +} + +export function assertInlineMaterializationSize(size: number, maxBytes?: number): void { + const limit = maxBytes ?? MAX_INLINE_MATERIALIZATION_BYTES + if (size > limit) { + throw new ExecutionResourceLimitError({ + resource: 'execution_payload_bytes', + attemptedBytes: size, + limitBytes: limit, + }) + } +} + +export function isValidLargeValueKey(ref: LargeValueRef): boolean { + return Boolean(ref.key && isLargeValueStorageKey(ref.key, ref.id, ref.executionId)) +} + +export function assertLargeValueRefAccess( + ref: LargeValueRef, + context: ExecutionMaterializationContext +): void { + if (!context.executionId) { + throw new Error('Large execution value requires an execution context.') + } + const allowedExecutionIds = new Set([ + context.executionId, + ...(context.largeValueExecutionIds ?? []), + ]) + + const parts = ref.key?.split('/') ?? [] + const [, workspaceId, workflowId, executionId] = parts + + if (!ref.key) { + if (ref.executionId && !allowedExecutionIds.has(ref.executionId)) { + throw new Error('Large execution value is not available in this execution.') + } + return + } + if (!context.workspaceId || !context.workflowId) { + throw new Error('Large execution value requires workspace and workflow context.') + } + const workflowScopeAllowed = + context.allowLargeValueWorkflowScope && + context.workspaceId === workspaceId && + context.workflowId === workflowId + if (ref.executionId && !allowedExecutionIds.has(ref.executionId) && !workflowScopeAllowed) { + throw new Error('Large execution value is not available in this execution.') + } + if (!allowedExecutionIds.has(executionId) && !workflowScopeAllowed) { + throw new Error('Large execution value is not available in this execution.') + } + if (context.workspaceId && workspaceId !== context.workspaceId) { + throw new Error('Large execution value is not available in this execution.') + } + if (context.workflowId && workflowId !== context.workflowId) { + throw new Error('Large execution value is not available in this execution.') + } +} + +export async function readLargeValueRefFromStorage( + ref: LargeValueRef, + options: MaterializeLargeValueOptions = {} +): Promise { + const log = getLogger(options) + if (!isLargeValueRef(ref) || !ref.key || !isValidLargeValueKey(ref)) { + return undefined + } + + assertLargeValueRefAccess(ref, options) + assertInlineMaterializationSize(ref.size, options.maxBytes) + + try { + const { StorageService } = await import('@/lib/uploads') + const buffer = await StorageService.downloadFile({ + key: ref.key, + context: 'execution', + }) + if (buffer.length > (options.maxBytes ?? MAX_INLINE_MATERIALIZATION_BYTES)) { + throw new ExecutionResourceLimitError({ + resource: 'execution_payload_bytes', + attemptedBytes: buffer.length, + limitBytes: options.maxBytes ?? MAX_INLINE_MATERIALIZATION_BYTES, + }) + } + return JSON.parse(buffer.toString('utf8')) + } catch (error) { + if (error instanceof ExecutionResourceLimitError) { + throw error + } + log.warn('Failed to materialize persisted large execution value', { + id: ref.id, + key: ref.key, + error: toError(error).message, + }) + return undefined + } +} + +function normalizeRange(buffer: Buffer, options: ReadUserFileContentOptions): Buffer { + const offset = Math.max(0, Math.floor(options.offset ?? 0)) + const maxLength = options.maxBytes ?? MAX_FUNCTION_INLINE_BYTES + const requestedLength = options.length === undefined ? maxLength : Math.floor(options.length) + const length = Math.max(0, Math.min(requestedLength, maxLength)) + return buffer.subarray(offset, offset + length) +} + +function getExecutionKeyParts(key: string): + | { + workspaceId: string + workflowId: string + executionId: string + } + | undefined { + const parts = key.split('/') + if (parts[0] !== 'execution' || parts.length < 5) { + return undefined + } + + return { + workspaceId: parts[1], + workflowId: parts[2], + executionId: parts[3], + } +} + +function assertExecutionFileScope(key: string, options: ExecutionMaterializationContext): void { + const parts = getExecutionKeyParts(key) + if (!parts) { + throw new Error('File is not available in this execution.') + } + + const allowedExecutionIds = new Set([ + options.executionId, + ...(options.largeValueExecutionIds ?? []), + ]) + const workflowScopeAllowed = + options.allowLargeValueWorkflowScope && + options.workspaceId === parts.workspaceId && + options.workflowId === parts.workflowId + if ( + !options.executionId || + (!allowedExecutionIds.has(parts.executionId) && !workflowScopeAllowed) + ) { + throw new Error('File is not available in this execution.') + } + + if (options.workspaceId && parts.workspaceId !== options.workspaceId) { + throw new Error('File is not available in this execution.') + } + + if (options.workflowId && parts.workflowId !== options.workflowId) { + throw new Error('File is not available in this execution.') + } +} + +function getVerifiedStorageContext(file: UserFile): StorageContext { + if (!file.key) { + throw new Error('File content requires a storage key.') + } + + const inferredContext = inferContextFromKey(file.key) + if (file.context && file.context !== inferredContext) { + throw new Error('File context does not match its storage key.') + } + + return inferredContext +} + +export async function assertUserFileContentAccess( + file: UserFile, + options: ExecutionMaterializationContext +): Promise { + const context = getVerifiedStorageContext(file) + + if (context === 'execution') { + assertExecutionFileScope(file.key, options) + } + + if (!options.userId) { + throw new Error('File access requires an authenticated user.') + } + + const { verifyFileAccess } = await import('@/app/api/files/authorization') + const hasAccess = await verifyFileAccess(file.key, options.userId, undefined, context, false) + if (!hasAccess) { + throw new Error('File is not available in this execution.') + } +} + +export async function readUserFileContent( + file: unknown, + options: ReadUserFileContentOptions +): Promise { + if (!isUserFileWithMetadata(file)) { + throw new Error('Expected a file object with metadata.') + } + + await assertUserFileContentAccess(file, options) + + const maxSourceBytes = options.maxSourceBytes ?? MAX_FUNCTION_FILE_BYTES + if (Number.isFinite(file.size) && file.size > maxSourceBytes) { + throw new ExecutionResourceLimitError({ + resource: 'execution_payload_bytes', + attemptedBytes: file.size, + limitBytes: maxSourceBytes, + }) + } + + let buffer: Buffer | null = null + const log = getLogger(options) + const requestId = options.requestId ?? 'unknown' + + buffer = await downloadFileFromStorage(file, requestId, log) + + if (!buffer) { + throw new Error(`File content for ${file.name} is unavailable.`) + } + if (buffer.length > maxSourceBytes) { + throw new ExecutionResourceLimitError({ + resource: 'execution_payload_bytes', + attemptedBytes: buffer.length, + limitBytes: maxSourceBytes, + }) + } + + const shouldSlice = + options.chunked || options.offset !== undefined || options.length !== undefined + const selected = shouldSlice ? normalizeRange(buffer, options) : buffer + assertInlineMaterializationSize(selected.length, options.maxBytes ?? MAX_FUNCTION_INLINE_BYTES) + + return options.encoding === 'base64' ? bufferToBase64(selected) : selected.toString('utf8') +} + +export function unavailableLargeValueError(ref: LargeValueRef): Error { + return getLargeValueMaterializationError(ref) +} diff --git a/apps/sim/lib/execution/payloads/serializer.test.ts b/apps/sim/lib/execution/payloads/serializer.test.ts new file mode 100644 index 00000000000..453c1637ece --- /dev/null +++ b/apps/sim/lib/execution/payloads/serializer.test.ts @@ -0,0 +1,129 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + getLargeValueMaterializationError, + isLargeValueRef, +} from '@/lib/execution/payloads/large-value-ref' +import { compactExecutionPayload } from '@/lib/execution/payloads/serializer' +import type { UserFile } from '@/executor/types' +import { navigatePath } from '@/executor/variables/resolvers/reference' + +const TEST_EXECUTION_CONTEXT = { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', +} + +describe('compactExecutionPayload', () => { + it('keeps small JSON payloads inline', async () => { + const value = { result: { id: 'event-1', text: 'hello' } } + + await expect(compactExecutionPayload(value, { thresholdBytes: 1024 })).resolves.toEqual(value) + }) + + it('strips UserFile base64 by default while preserving metadata', async () => { + const file: UserFile = { + id: 'file-1', + name: 'large.txt', + url: 'https://example.com/file', + size: 11 * 1024 * 1024, + type: 'text/plain', + key: 'execution/workflow/execution/large.txt', + context: 'execution', + base64: 'Zm9v', + } + + const compacted = await compactExecutionPayload( + { event: { files: [file] } }, + { thresholdBytes: 1024 } + ) + + expect(compacted).toEqual({ + event: { + files: [ + { + id: 'file-1', + name: 'large.txt', + url: 'https://example.com/file', + size: 11 * 1024 * 1024, + type: 'text/plain', + key: 'execution/workflow/execution/large.txt', + context: 'execution', + }, + ], + }, + }) + }) + + it('stores oversized arrays as refs and allows nested path navigation in-process', async () => { + const results = Array.from({ length: 100 }, (_, index) => [{ event: { id: `event-${index}` } }]) + const compacted = await compactExecutionPayload( + { results }, + { thresholdBytes: 256, ...TEST_EXECUTION_CONTEXT } + ) + + expect(isLargeValueRef(compacted.results)).toBe(true) + expect( + navigatePath(compacted, ['results', '1', '0', 'event', 'id'], { + executionContext: TEST_EXECUTION_CONTEXT, + }) + ).toBe('event-1') + }) + + it('does not double-spill existing refs', async () => { + const compacted = await compactExecutionPayload( + { results: [[{ payload: 'x'.repeat(2048) }]] }, + { thresholdBytes: 256 } + ) + + const compactedAgain = await compactExecutionPayload(compacted, { thresholdBytes: 256 }) + + expect(compactedAgain).toEqual(compacted) + }) + + it('rejects durable compaction when storage context is incomplete', async () => { + await expect( + compactExecutionPayload( + { payload: 'x'.repeat(2048) }, + { thresholdBytes: 256, requireDurable: true } + ) + ).rejects.toThrow('Cannot persist large execution value') + }) + + it('does not treat loosely marker-shaped user data as a large-value ref', () => { + expect( + isLargeValueRef({ + __simLargeValueRef: true, + id: 'user-supplied', + }) + ).toBe(false) + }) + + it('rejects ref-shaped user data with non-execution storage keys', () => { + expect( + isLargeValueRef({ + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'object', + size: 1024, + key: 'https://example.com/large-value-lv_ABCDEFGHIJKL.json', + }) + ).toBe(false) + }) + + it('omits opaque ref IDs from user-facing materialization errors', () => { + const error = getLargeValueMaterializationError({ + __simLargeValueRef: true, + version: 1, + id: 'lv_CQcekP8gSJI5', + kind: 'string', + size: 23_259_101, + }) + + expect(error.message).toContain('This execution value is too large to inline (22.2 MB)') + expect(error.message).not.toContain('lv_CQcekP8gSJI5') + }) +}) diff --git a/apps/sim/lib/execution/payloads/serializer.ts b/apps/sim/lib/execution/payloads/serializer.ts new file mode 100644 index 00000000000..d892b2a3226 --- /dev/null +++ b/apps/sim/lib/execution/payloads/serializer.ts @@ -0,0 +1,162 @@ +import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' +import { + isLargeValueRef, + LARGE_VALUE_THRESHOLD_BYTES, +} from '@/lib/execution/payloads/large-value-ref' +import { type LargeValueStoreContext, storeLargeValue } from '@/lib/execution/payloads/store' +import type { BlockLog } from '@/executor/types' + +export interface CompactExecutionPayloadOptions extends LargeValueStoreContext { + thresholdBytes?: number + preserveUserFileBase64?: boolean + preserveRoot?: boolean +} + +interface CompactState { + seen: WeakSet +} + +function getJsonAndSize(value: unknown): { json: string; size: number } | null { + try { + const json = JSON.stringify(value) + if (json === undefined) { + return null + } + return { + json, + size: Buffer.byteLength(json, 'utf8'), + } + } catch { + return null + } +} + +function stripUserFileBase64(value: T): Omit { + const { base64: _base64, ...rest } = value + return rest +} + +async function compactValue( + value: unknown, + options: CompactExecutionPayloadOptions, + state: CompactState, + depth = 0 +): Promise { + if (!value || typeof value !== 'object') { + const measured = getJsonAndSize(value) + if (measured && measured.size > (options.thresholdBytes ?? LARGE_VALUE_THRESHOLD_BYTES)) { + return options.preserveRoot && depth === 0 + ? value + : storeLargeValue(value, measured.json, measured.size, options) + } + return value + } + + if (isLargeValueRef(value)) { + return value + } + + if (isUserFileWithMetadata(value) && !options.preserveUserFileBase64) { + return stripUserFileBase64(value) + } + + if (state.seen.has(value)) { + return value + } + state.seen.add(value) + + const compacted = Array.isArray(value) + ? await Promise.all(value.map((item) => compactValue(item, options, state, depth + 1))) + : Object.fromEntries( + await Promise.all( + Object.entries(value).map(async ([key, entryValue]) => [ + key, + key === 'finalBlockLogs' && Array.isArray(entryValue) + ? await compactBlockLogs(entryValue as BlockLog[], options) + : await compactValue(entryValue, options, state, depth + 1), + ]) + ) + ) + + const measured = getJsonAndSize(compacted) + if (measured && measured.size > (options.thresholdBytes ?? LARGE_VALUE_THRESHOLD_BYTES)) { + return options.preserveRoot && depth === 0 + ? compacted + : storeLargeValue(compacted, measured.json, measured.size, options) + } + + return compacted +} + +async function forceStoreValue( + value: unknown, + options: CompactExecutionPayloadOptions +): Promise { + if (isLargeValueRef(value)) { + return value + } + const measured = getJsonAndSize(value) + if (!measured) { + return value + } + return storeLargeValue(value, measured.json, measured.size, options) +} + +export async function compactExecutionPayload( + value: T, + options: CompactExecutionPayloadOptions = {} +): Promise { + return (await compactValue(value, options, { seen: new WeakSet() })) as T +} + +/** + * Compacts subflow result aggregates while preserving indexable `results`. + */ +export async function compactSubflowResults( + results: T[], + options: CompactExecutionPayloadOptions = {} +): Promise { + const entryOptions = { ...options, preserveRoot: false } + let compactedResults = (await Promise.all( + results.map((result) => compactExecutionPayload(result, entryOptions)) + )) as T[] + + const aggregate = getJsonAndSize({ results: compactedResults }) + if (aggregate && aggregate.size <= (options.thresholdBytes ?? LARGE_VALUE_THRESHOLD_BYTES)) { + return compactedResults + } + + compactedResults = (await Promise.all( + compactedResults.map((result) => forceStoreValue(result, options)) + )) as T[] + + return compactedResults +} + +export async function compactBlockLogs( + logs: BlockLog[] | undefined, + options: CompactExecutionPayloadOptions = {} +): Promise { + if (!logs) { + return logs + } + + return Promise.all( + logs.map(async (log) => { + const compactedLog = { ...log } + if ('input' in compactedLog) { + compactedLog.input = await compactExecutionPayload(compactedLog.input, options) + } + if ('output' in compactedLog) { + compactedLog.output = await compactExecutionPayload(compactedLog.output, options) + } + if ('childTraceSpans' in compactedLog) { + compactedLog.childTraceSpans = await compactExecutionPayload( + compactedLog.childTraceSpans, + options + ) + } + return compactedLog + }) + ) +} diff --git a/apps/sim/lib/execution/payloads/store.test.ts b/apps/sim/lib/execution/payloads/store.test.ts new file mode 100644 index 00000000000..089f8284f07 --- /dev/null +++ b/apps/sim/lib/execution/payloads/store.test.ts @@ -0,0 +1,461 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + cacheLargeValue, + clearLargeValueCacheForTests, + materializeLargeValueRefSync, +} from '@/lib/execution/payloads/cache' +import { + MAX_DURABLE_LARGE_VALUE_BYTES, + readLargeValueRefFromStorage, + readUserFileContent, +} from '@/lib/execution/payloads/materialization.server' +import { materializeLargeValueRef, storeLargeValue } from '@/lib/execution/payloads/store' +import { EXECUTION_RESOURCE_LIMIT_CODE } from '@/lib/execution/resource-errors' + +const { mockDownloadFile, mockUploadFile, mockVerifyFileAccess } = vi.hoisted(() => ({ + mockDownloadFile: vi.fn(), + mockUploadFile: vi.fn(), + mockVerifyFileAccess: vi.fn(), +})) + +vi.mock('@/lib/uploads', () => ({ + StorageService: { + uploadFile: mockUploadFile, + downloadFile: mockDownloadFile, + }, +})) + +vi.mock('@/app/api/files/authorization', () => ({ + verifyFileAccess: mockVerifyFileAccess, +})) + +describe('large execution payload store', () => { + beforeEach(() => { + vi.clearAllMocks() + clearLargeValueCacheForTests() + mockUploadFile.mockImplementation(async ({ customKey }) => ({ key: customKey })) + mockVerifyFileAccess.mockResolvedValue(true) + }) + + it('stores oversized JSON in execution object storage and returns a small ref', async () => { + const value = { payload: 'x'.repeat(2048) } + const json = JSON.stringify(value) + + const ref = await storeLargeValue(value, json, Buffer.byteLength(json, 'utf8'), { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + userId: 'user-1', + requireDurable: true, + }) + + expect(ref).toMatchObject({ + __simLargeValueRef: true, + version: 1, + kind: 'object', + size: Buffer.byteLength(json, 'utf8'), + executionId: 'execution-1', + }) + expect(ref.key).toBe(`execution/workspace-1/workflow-1/execution-1/large-value-${ref.id}.json`) + expect(mockUploadFile).toHaveBeenCalledWith( + expect.objectContaining({ + contentType: 'application/json', + context: 'execution', + preserveKey: true, + customKey: ref.key, + }) + ) + }) + + it('fails durable writes before producing refs when execution context is missing', async () => { + const value = { payload: 'x'.repeat(2048) } + const json = JSON.stringify(value) + + await expect( + storeLargeValue(value, json, Buffer.byteLength(json, 'utf8'), { requireDurable: true }) + ).rejects.toThrow('Cannot persist large execution value') + + expect(mockUploadFile).not.toHaveBeenCalled() + }) + + it('fails durable writes when storage upload fails', async () => { + const value = { payload: 'x'.repeat(2048) } + const json = JSON.stringify(value) + mockUploadFile.mockRejectedValueOnce(new Error('storage down')) + + await expect( + storeLargeValue(value, json, Buffer.byteLength(json, 'utf8'), { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + requireDurable: true, + }) + ).rejects.toThrow('Failed to persist large execution value: storage down') + }) + + it('materializes object-storage refs through the server helper', async () => { + mockDownloadFile.mockResolvedValueOnce(Buffer.from(JSON.stringify({ ok: true }), 'utf8')) + + await expect( + materializeLargeValueRef( + { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'object', + size: 11, + key: 'execution/workflow-1/workflow-2/execution-1/large-value-lv_ABCDEFGHIJKL.json', + executionId: 'execution-1', + }, + { + workspaceId: 'workflow-1', + workflowId: 'workflow-2', + executionId: 'execution-1', + } + ) + ).resolves.toEqual({ ok: true }) + }) + + it('bounds durable large-value writes', async () => { + const size = MAX_DURABLE_LARGE_VALUE_BYTES + 1 + + await expect( + storeLargeValue('x', JSON.stringify('x'), size, { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + requireDurable: true, + }) + ).rejects.toMatchObject({ code: EXECUTION_RESOURCE_LIMIT_CODE }) + }) + + it('bounds explicit server-side materialization', async () => { + await expect( + readLargeValueRefFromStorage( + { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'object', + size: 2048, + key: 'execution/workflow-1/workflow-2/execution-1/large-value-lv_ABCDEFGHIJKL.json', + executionId: 'execution-1', + }, + { + workspaceId: 'workflow-1', + workflowId: 'workflow-2', + executionId: 'execution-1', + maxBytes: 1024, + } + ) + ).rejects.toMatchObject({ code: EXECUTION_RESOURCE_LIMIT_CODE }) + }) + + it('does not materialize durable refs without caller execution context', async () => { + await expect( + materializeLargeValueRef({ + __simLargeValueRef: true, + version: 1, + id: 'lv_NOCTXVALUE12', + kind: 'object', + size: 11, + key: 'execution/workflow-1/workflow-2/execution-1/large-value-lv_NOCTXVALUE12.json', + executionId: 'execution-1', + }) + ).resolves.toBeUndefined() + + expect(mockDownloadFile).not.toHaveBeenCalled() + }) + + it('checks caller execution context before returning cached large values', async () => { + const value = { payload: 'cached' } + const json = JSON.stringify(value) + const ref = await storeLargeValue(value, json, Buffer.byteLength(json, 'utf8'), { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + userId: 'user-1', + requireDurable: true, + }) + + await expect( + materializeLargeValueRef(ref, { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'other-execution', + userId: 'user-1', + }) + ).rejects.toThrow('Large execution value is not available in this execution.') + }) + + it('rejects durable refs whose key does not match caller execution context', async () => { + await expect( + readLargeValueRefFromStorage( + { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'object', + size: 11, + key: 'execution/workflow-1/workflow-2/execution-1/large-value-lv_ABCDEFGHIJKL.json', + executionId: 'execution-1', + }, + { workspaceId: 'workflow-1', workflowId: 'workflow-2', executionId: 'other-execution' } + ) + ).rejects.toThrow('Large execution value is not available in this execution.') + + expect(mockDownloadFile).not.toHaveBeenCalled() + }) + + it('allows prior-execution durable refs only when workflow-scoped reads are explicitly enabled', async () => { + mockDownloadFile.mockResolvedValueOnce(Buffer.from(JSON.stringify({ ok: true }), 'utf8')) + + await expect( + readLargeValueRefFromStorage( + { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'object', + size: 11, + key: 'execution/workspace-1/workflow-1/source-execution/large-value-lv_ABCDEFGHIJKL.json', + executionId: 'source-execution', + }, + { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'resume-execution', + allowLargeValueWorkflowScope: true, + } + ) + ).resolves.toEqual({ ok: true }) + }) + + it('does not materialize forged keyless refs from another cached execution', () => { + cacheLargeValue('lv_FORGEDCACHE1', { secret: true }, 16, { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'source-execution', + }) + + const forged = { + __simLargeValueRef: true, + version: 1, + id: 'lv_FORGEDCACHE1', + kind: 'object', + size: 16, + executionId: 'other-execution', + } as const + + expect( + materializeLargeValueRefSync(forged, { + workspaceId: 'workspace-2', + workflowId: 'workflow-2', + executionId: 'other-execution', + }) + ).toBeUndefined() + }) + + it('does not evict unrecoverable in-memory refs for recoverable cache entries', () => { + const scope = { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + } + const unrecoverableId = 'lv_UNRECOVER001' + const unrecoverableRef = { + __simLargeValueRef: true, + version: 1, + id: unrecoverableId, + kind: 'object', + size: 200 * 1024 * 1024, + executionId: scope.executionId, + } as const + + expect(cacheLargeValue(unrecoverableId, { retained: true }, unrecoverableRef.size, scope)).toBe( + true + ) + expect( + cacheLargeValue('lv_RECOVER00001', { recoverable: true }, 70 * 1024 * 1024, scope, { + recoverable: true, + }) + ).toBe(false) + expect(materializeLargeValueRefSync(unrecoverableRef, scope)).toEqual({ retained: true }) + }) + + it('materializes keyless cached refs through the async helper', async () => { + const scope = { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + } + const ref = { + __simLargeValueRef: true, + version: 1, + id: 'lv_KEYLESSCACHE', + kind: 'object', + size: 32, + executionId: scope.executionId, + } as const + cacheLargeValue(ref.id, { retained: true }, ref.size, scope) + + await expect(materializeLargeValueRef(ref, scope)).resolves.toEqual({ retained: true }) + expect(mockDownloadFile).not.toHaveBeenCalled() + }) + + it('enforces maxBytes before returning cached refs', async () => { + const scope = { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + } + const ref = { + __simLargeValueRef: true, + version: 1, + id: 'lv_CACHEDMAXBYTE', + kind: 'object', + size: 2048, + executionId: scope.executionId, + } as const + cacheLargeValue(ref.id, { retained: true }, ref.size, scope) + + await expect(materializeLargeValueRef(ref, { ...scope, maxBytes: 1024 })).rejects.toMatchObject( + { + code: EXECUTION_RESOURCE_LIMIT_CODE, + } + ) + expect(mockDownloadFile).not.toHaveBeenCalled() + }) + + it('rejects durable refs when caller omits workspace and workflow context', async () => { + await expect( + readLargeValueRefFromStorage( + { + __simLargeValueRef: true, + version: 1, + id: 'lv_ABCDEFGHIJKL', + kind: 'object', + size: 11, + key: 'execution/workflow-1/workflow-2/execution-1/large-value-lv_ABCDEFGHIJKL.json', + executionId: 'execution-1', + }, + { executionId: 'execution-1' } + ) + ).rejects.toThrow('Large execution value requires workspace and workflow context.') + + expect(mockDownloadFile).not.toHaveBeenCalled() + }) + + it('rejects execution files with forged public contexts before storage download', async () => { + await expect( + readUserFileContent( + { + id: 'file_1', + name: 'secret.txt', + url: '/api/files/serve/execution/workspace-1/workflow-1/execution-1/secret.txt', + key: 'execution/workspace-1/workflow-1/execution-1/secret.txt', + context: 'profile-pictures', + size: 32, + type: 'text/plain', + }, + { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + userId: 'user-1', + encoding: 'text', + } + ) + ).rejects.toThrow('File context does not match its storage key.') + + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + expect(mockDownloadFile).not.toHaveBeenCalled() + }) + + it('rejects URL-only file objects instead of reading internal URLs directly', async () => { + await expect( + readUserFileContent( + { + id: 'file_1', + name: 'secret.txt', + url: '/api/files/serve/execution/workspace-1/workflow-1/execution-1/secret.txt?context=execution', + key: '', + size: 32, + type: 'text/plain', + }, + { + workspaceId: 'workspace-1', + workflowId: 'workflow-1', + executionId: 'execution-1', + userId: 'user-1', + encoding: 'text', + } + ) + ).rejects.toThrow('File content requires a storage key.') + + expect(mockVerifyFileAccess).not.toHaveBeenCalled() + expect(mockDownloadFile).not.toHaveBeenCalled() + }) + + it('throws instead of truncating non-chunked file reads over the inline cap', async () => { + const workspaceId = '11111111-1111-4111-8111-111111111111' + const workflowId = '22222222-2222-4222-8222-222222222222' + const executionId = '33333333-3333-4333-8333-333333333333' + mockDownloadFile.mockResolvedValueOnce(Buffer.from('hello world', 'utf8')) + + await expect( + readUserFileContent( + { + id: 'file_1', + name: 'hello.txt', + url: `/api/files/serve/execution/${workspaceId}/${workflowId}/${executionId}/hello.txt`, + key: `execution/${workspaceId}/${workflowId}/${executionId}/hello.txt`, + context: 'execution', + size: 11, + type: 'text/plain', + }, + { + workspaceId, + workflowId, + executionId, + userId: 'user-1', + encoding: 'text', + maxBytes: 5, + } + ) + ).rejects.toMatchObject({ code: EXECUTION_RESOURCE_LIMIT_CODE }) + }) + + it('allows explicit chunked file reads to slice within the inline cap', async () => { + const workspaceId = '11111111-1111-4111-8111-111111111111' + const workflowId = '22222222-2222-4222-8222-222222222222' + const executionId = '33333333-3333-4333-8333-333333333333' + mockDownloadFile.mockResolvedValueOnce(Buffer.from('hello world', 'utf8')) + + await expect( + readUserFileContent( + { + id: 'file_1', + name: 'hello.txt', + url: `/api/files/serve/execution/${workspaceId}/${workflowId}/${executionId}/hello.txt`, + key: `execution/${workspaceId}/${workflowId}/${executionId}/hello.txt`, + context: 'execution', + size: 11, + type: 'text/plain', + }, + { + workspaceId, + workflowId, + executionId, + userId: 'user-1', + encoding: 'text', + maxBytes: 5, + chunked: true, + } + ) + ).resolves.toBe('hello') + }) +}) diff --git a/apps/sim/lib/execution/payloads/store.ts b/apps/sim/lib/execution/payloads/store.ts new file mode 100644 index 00000000000..cf1eb00367b --- /dev/null +++ b/apps/sim/lib/execution/payloads/store.ts @@ -0,0 +1,180 @@ +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { generateShortId } from '@sim/utils/id' +import { cacheLargeValue, materializeLargeValueRefSync } from '@/lib/execution/payloads/cache' +import { + LARGE_VALUE_REF_VERSION, + type LargeValueKind, + type LargeValueRef, +} from '@/lib/execution/payloads/large-value-ref' +import { + assertDurableLargeValueSize, + assertInlineMaterializationSize, + assertLargeValueRefAccess, + isValidLargeValueKey, + readLargeValueRefFromStorage, +} from '@/lib/execution/payloads/materialization.server' +import { generateExecutionFileKey } from '@/lib/uploads/contexts/execution/utils' + +const logger = createLogger('LargeExecutionPayloadStore') + +export interface LargeValueStoreContext { + workspaceId?: string + workflowId?: string + executionId?: string + largeValueExecutionIds?: string[] + allowLargeValueWorkflowScope?: boolean + userId?: string + requireDurable?: boolean + maxBytes?: number +} + +function getKind(value: unknown): LargeValueKind { + if (typeof value === 'string') return 'string' + if (Array.isArray(value)) return 'array' + if (value && typeof value === 'object') return 'object' + return 'json' +} + +function getPreview(value: unknown): unknown { + if (typeof value === 'string') { + return value.length > 256 ? `${value.slice(0, 256)}...` : value + } + if (Array.isArray(value)) { + return { length: value.length } + } + if (value && typeof value === 'object') { + return { keys: Object.keys(value).slice(0, 20) } + } + return value +} + +async function persistValue( + id: string, + json: string, + context: LargeValueStoreContext +): Promise { + const { workspaceId, workflowId, executionId, userId } = context + if (!workspaceId || !workflowId || !executionId) { + if (context.requireDurable) { + throw new Error( + 'Cannot persist large execution value without workspace, workflow, and execution IDs' + ) + } + return undefined + } + + const key = generateExecutionFileKey( + { workspaceId, workflowId, executionId }, + `large-value-${id}.json` + ) + + try { + const { StorageService } = await import('@/lib/uploads') + const fileInfo = await StorageService.uploadFile({ + file: Buffer.from(json, 'utf8'), + fileName: key, + contentType: 'application/json', + context: 'execution', + preserveKey: true, + customKey: key, + metadata: { + originalName: `large-value-${id}.json`, + uploadedAt: new Date().toISOString(), + purpose: 'execution-large-value', + workspaceId, + ...(userId ? { userId } : {}), + }, + }) + return fileInfo.key + } catch (error) { + if (context.requireDurable) { + throw new Error(`Failed to persist large execution value: ${toError(error).message}`) + } + logger.warn('Failed to persist large execution value, keeping in memory only', { + id, + error: toError(error).message, + }) + return undefined + } +} + +export async function storeLargeValue( + value: unknown, + json: string, + size: number, + context: LargeValueStoreContext +): Promise { + assertDurableLargeValueSize(size) + const id = `lv_${generateShortId(12)}` + const key = await persistValue(id, json, context) + const cached = cacheLargeValue(id, value, size, context, { recoverable: Boolean(key) }) + if (!key && !cached) { + throw new Error('Cannot retain large execution value without durable storage') + } + + return { + __simLargeValueRef: true, + version: LARGE_VALUE_REF_VERSION, + id, + kind: getKind(value), + size, + key, + executionId: context.executionId, + preview: getPreview(value), + } +} + +export async function materializeLargeValueRef( + ref: LargeValueRef, + context?: LargeValueStoreContext +): Promise { + if (!context?.executionId) { + return undefined + } + + assertLargeValueRefAccess(ref, context) + assertInlineMaterializationSize(ref.size, context.maxBytes) + + const cached = materializeLargeValueRefSync(ref, context) + if (cached !== undefined) { + return cached + } + + if (!ref.key || !isValidLargeValueKey(ref)) { + return undefined + } + + try { + const value = await readLargeValueRefFromStorage(ref, { + workspaceId: context.workspaceId, + workflowId: context.workflowId, + executionId: context.executionId, + largeValueExecutionIds: context.largeValueExecutionIds, + allowLargeValueWorkflowScope: context.allowLargeValueWorkflowScope, + userId: context.userId, + maxBytes: context.maxBytes ?? ref.size, + }) + if (value === undefined) { + return undefined + } + cacheLargeValue( + ref.id, + value, + ref.size, + { + ...context, + executionId: ref.executionId ?? context.executionId, + }, + { recoverable: true } + ) + return value + } catch (error) { + logger.warn('Failed to materialize persisted large execution value', { + id: ref.id, + key: ref.key, + error, + }) + return undefined + } +} diff --git a/apps/sim/lib/execution/redis-budget.server.ts b/apps/sim/lib/execution/redis-budget.server.ts new file mode 100644 index 00000000000..ddf58b1772d --- /dev/null +++ b/apps/sim/lib/execution/redis-budget.server.ts @@ -0,0 +1,138 @@ +import { createLogger, type Logger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import type { getRedisClient } from '@/lib/core/config/redis' +import { ExecutionResourceLimitError } from '@/lib/execution/resource-errors' + +type RedisClient = NonNullable> + +const logger = createLogger('ExecutionRedisBudget') +const REDIS_BUDGET_PREFIX = 'execution:redis-budget:' +const MAX_SINGLE_REDIS_WRITE_BYTES = 8 * 1024 * 1024 +const MAX_EXECUTION_REDIS_BYTES = 64 * 1024 * 1024 +const MAX_USER_REDIS_BYTES = 256 * 1024 * 1024 +const REDIS_BUDGET_TTL_SECONDS = 60 * 60 + +const RESERVE_REDIS_BYTES_SCRIPT = ` +local bytes = tonumber(ARGV[1]) +local execution_limit = tonumber(ARGV[2]) +local user_limit = tonumber(ARGV[3]) +local ttl_seconds = tonumber(ARGV[4]) +local execution_current = tonumber(redis.call('GET', KEYS[1]) or '0') +if execution_limit > 0 and execution_current + bytes > execution_limit then + return {0, 'execution_redis_bytes', execution_current} +end +local user_current = 0 +if #KEYS >= 2 then + user_current = tonumber(redis.call('GET', KEYS[2]) or '0') + if user_limit > 0 and user_current + bytes > user_limit then + return {0, 'user_redis_bytes', user_current} + end +end +redis.call('INCRBY', KEYS[1], bytes) +redis.call('EXPIRE', KEYS[1], ttl_seconds) +if #KEYS >= 2 then + redis.call('INCRBY', KEYS[2], bytes) + redis.call('EXPIRE', KEYS[2], ttl_seconds) +end +return {1, 'ok', execution_current + bytes, user_current + bytes} +` + +const RELEASE_REDIS_BYTES_SCRIPT = ` +local bytes = tonumber(ARGV[1]) +for i = 1, #KEYS do + local next_value = redis.call('DECRBY', KEYS[i], bytes) + if next_value <= 0 then + redis.call('DEL', KEYS[i]) + end +end +return 1 +` + +export type ExecutionRedisBudgetCategory = 'event_buffer' | 'base64_cache' + +export interface ExecutionRedisBudgetReservation { + executionId: string + userId?: string + category: ExecutionRedisBudgetCategory + bytes: number + operation: string + logger?: Logger +} + +export function getExecutionRedisBudgetLimits() { + return { + maxSingleWriteBytes: MAX_SINGLE_REDIS_WRITE_BYTES, + maxExecutionBytes: MAX_EXECUTION_REDIS_BYTES, + maxUserBytes: MAX_USER_REDIS_BYTES, + ttlSeconds: REDIS_BUDGET_TTL_SECONDS, + } +} + +export function getExecutionRedisBudgetKeys( + reservation: ExecutionRedisBudgetReservation +): string[] { + const keys = [`${REDIS_BUDGET_PREFIX}execution:${reservation.executionId}`] + if (reservation.userId) { + keys.push(`${REDIS_BUDGET_PREFIX}user:${reservation.userId}`) + } + return keys +} + +export async function reserveExecutionRedisBytes( + redis: RedisClient, + reservation: ExecutionRedisBudgetReservation +): Promise { + if (reservation.bytes <= 0) return + + const limits = getExecutionRedisBudgetLimits() + if (reservation.bytes > limits.maxSingleWriteBytes) { + throw new ExecutionResourceLimitError({ + resource: 'redis_key_bytes', + attemptedBytes: reservation.bytes, + limitBytes: limits.maxSingleWriteBytes, + }) + } + + const keys = getExecutionRedisBudgetKeys(reservation) + const result = (await redis.eval( + RESERVE_REDIS_BYTES_SCRIPT, + keys.length, + ...keys, + reservation.bytes, + limits.maxExecutionBytes, + limits.maxUserBytes, + limits.ttlSeconds + )) as [number, string, number | string | null] + + const [allowed, resource, current] = result + if (allowed === 1) return + + throw new ExecutionResourceLimitError({ + resource: resource === 'user_redis_bytes' ? 'user_redis_bytes' : 'execution_redis_bytes', + attemptedBytes: reservation.bytes, + currentBytes: Number(current ?? 0), + limitBytes: resource === 'user_redis_bytes' ? limits.maxUserBytes : limits.maxExecutionBytes, + }) +} + +export async function releaseExecutionRedisBytes( + redis: RedisClient, + reservation: ExecutionRedisBudgetReservation +): Promise { + if (reservation.bytes <= 0) return + + try { + const keys = getExecutionRedisBudgetKeys(reservation) + await redis.eval(RELEASE_REDIS_BYTES_SCRIPT, keys.length, ...keys, reservation.bytes) + } catch (error) { + const log = reservation.logger ?? logger + log.warn('Failed to release execution Redis budget reservation', { + executionId: reservation.executionId, + userId: reservation.userId, + category: reservation.category, + operation: reservation.operation, + bytes: reservation.bytes, + error: toError(error).message, + }) + } +} diff --git a/apps/sim/lib/execution/resource-errors.ts b/apps/sim/lib/execution/resource-errors.ts new file mode 100644 index 00000000000..3cd2f61bad9 --- /dev/null +++ b/apps/sim/lib/execution/resource-errors.ts @@ -0,0 +1,45 @@ +export const EXECUTION_RESOURCE_LIMIT_CODE = 'execution_resource_limit_exceeded' as const + +export type ExecutionResourceLimitResource = + | 'redis_key_bytes' + | 'execution_redis_bytes' + | 'user_redis_bytes' + | 'execution_payload_bytes' + +export interface ExecutionResourceLimitDetails { + resource: ExecutionResourceLimitResource + attemptedBytes: number + limitBytes: number + currentBytes?: number + statusCode?: number +} + +export class ExecutionResourceLimitError extends Error { + readonly code = EXECUTION_RESOURCE_LIMIT_CODE + readonly statusCode: number + readonly resource: ExecutionResourceLimitResource + readonly attemptedBytes: number + readonly limitBytes: number + readonly currentBytes?: number + + constructor(details: ExecutionResourceLimitDetails) { + super('Execution memory limit exceeded. Reduce payload size and try again.') + this.name = 'ExecutionResourceLimitError' + this.resource = details.resource + this.attemptedBytes = details.attemptedBytes + this.limitBytes = details.limitBytes + this.currentBytes = details.currentBytes + this.statusCode = details.statusCode ?? (details.resource === 'user_redis_bytes' ? 429 : 413) + } +} + +export function isExecutionResourceLimitError( + error: unknown +): error is ExecutionResourceLimitError { + return ( + error instanceof ExecutionResourceLimitError || + (typeof error === 'object' && + error !== null && + (error as { code?: unknown }).code === EXECUTION_RESOURCE_LIMIT_CODE) + ) +} diff --git a/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts b/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts index 6c237668c73..4665b6fc228 100644 --- a/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts +++ b/apps/sim/lib/uploads/contexts/execution/execution-file-manager.ts @@ -114,7 +114,6 @@ export async function uploadExecutionFile( url: presignedUrl, key: fileInfo.key, context: 'execution', - base64: fileBuffer.toString('base64'), } logger.info(`Successfully uploaded execution file: ${fileName} (${fileBuffer.length} bytes)`, { diff --git a/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts b/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts new file mode 100644 index 00000000000..00fe9ab75d4 --- /dev/null +++ b/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts @@ -0,0 +1,245 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + cleanupExecutionBase64Cache, + hydrateUserFilesWithBase64, +} from '@/lib/uploads/utils/user-file-base64.server' +import type { UserFile } from '@/executor/types' + +const { mockDownloadFile, mockGetRedisClient, mockRedis, mockVerifyFileAccess } = vi.hoisted(() => { + const mockRedis = { + get: vi.fn(), + set: vi.fn(), + hget: vi.fn(), + hset: vi.fn(), + hgetall: vi.fn(), + expire: vi.fn(), + scan: vi.fn(), + del: vi.fn(), + eval: vi.fn(), + } + return { + mockDownloadFile: vi.fn(), + mockGetRedisClient: vi.fn(), + mockRedis, + mockVerifyFileAccess: vi.fn(), + } +}) + +vi.mock('@/lib/core/config/redis', () => ({ + getRedisClient: mockGetRedisClient, +})) + +vi.mock('@/lib/uploads', () => ({ + StorageService: { + downloadFile: mockDownloadFile, + }, +})) + +vi.mock('@/lib/uploads/contexts/execution/execution-file-manager', () => ({ + downloadExecutionFile: mockDownloadFile, +})) + +vi.mock('@/lib/uploads/utils/file-utils.server', () => ({ + downloadFileFromStorage: mockDownloadFile, +})) + +vi.mock('@/app/api/files/authorization', () => ({ + verifyFileAccess: mockVerifyFileAccess, +})) + +describe('hydrateUserFilesWithBase64', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetRedisClient.mockReturnValue(null) + mockRedis.get.mockResolvedValue(null) + mockRedis.set.mockResolvedValue('OK') + mockRedis.hget.mockResolvedValue(null) + mockRedis.hset.mockResolvedValue(1) + mockRedis.hgetall.mockResolvedValue({}) + mockRedis.expire.mockResolvedValue(1) + mockRedis.scan.mockResolvedValue(['0', []]) + mockRedis.del.mockResolvedValue(1) + mockRedis.eval.mockResolvedValue([1, 'ok', 0, 0]) + mockVerifyFileAccess.mockResolvedValue(true) + }) + + it('strips existing base64 when it exceeds maxBytes', async () => { + const file: UserFile = { + id: 'file-1', + name: 'large.txt', + key: 'execution/workspace/workflow/execution/large.txt', + url: 'https://example.com/large.txt', + size: 5, + type: 'text/plain', + context: 'execution', + base64: Buffer.from('hello').toString('base64'), + } + + const hydrated = await hydrateUserFilesWithBase64({ file }, { maxBytes: 1 }) + + expect(hydrated.file).not.toHaveProperty('base64') + }) + + it('keeps existing base64 when it is within maxBytes', async () => { + const base64 = Buffer.from('hello').toString('base64') + const file: UserFile = { + id: 'file-1', + name: 'small.txt', + key: 'execution/workspace/workflow/execution/small.txt', + url: 'https://example.com/small.txt', + size: 5, + type: 'text/plain', + context: 'execution', + base64, + } + + const hydrated = await hydrateUserFilesWithBase64({ file }, { maxBytes: 10 }) + + expect(hydrated.file.base64).toBe(base64) + }) + + it('does not hydrate URL-only internal file objects', async () => { + const file: UserFile = { + id: 'file-1', + name: 'private.txt', + key: '', + url: '/api/files/serve/execution/workspace/workflow/execution/private.txt?context=execution', + size: 5, + type: 'text/plain', + } + + const hydrated = await hydrateUserFilesWithBase64({ file }, { maxBytes: 10, userId: 'user-1' }) + + expect(hydrated.file).not.toHaveProperty('base64') + }) + + it('hydrates prior-execution files when workflow-scoped reads are enabled', async () => { + mockDownloadFile.mockResolvedValueOnce(Buffer.from('hello', 'utf8')) + const file: UserFile = { + id: 'file-1', + name: 'prior.txt', + key: 'execution/workspace/workflow/source-execution/prior.txt', + url: '/api/files/serve/execution/workspace/workflow/source-execution/prior.txt?context=execution', + size: 5, + type: 'text/plain', + context: 'execution', + } + + const hydrated = await hydrateUserFilesWithBase64( + { file }, + { + workspaceId: 'workspace', + workflowId: 'workflow', + executionId: 'resume-execution', + allowLargeValueWorkflowScope: true, + userId: 'user-1', + maxBytes: 10, + } + ) + + expect(hydrated.file.base64).toBe(Buffer.from('hello').toString('base64')) + }) + + it('releases reserved Redis budget when cleaning up execution cache entries', async () => { + mockGetRedisClient.mockReturnValue(mockRedis) + const rawEntry = JSON.stringify({ bytes: 12, userId: 'user-1' }) + mockRedis.hgetall.mockResolvedValueOnce({ + 'key:file-1': rawEntry, + }) + mockRedis.eval.mockImplementation(async (script: string, ...args: unknown[]) => { + if (script.includes('HGET') && script.includes('HDEL') && script.includes('DECRBY')) { + expect(args).toEqual([ + 4, + 'user-file:base64-budget:exec:exec-1', + 'user-file:base64:exec:exec-1:key:file-1', + 'execution:redis-budget:execution:exec-1', + 'execution:redis-budget:user:user-1', + 'key:file-1', + rawEntry, + 12, + 60 * 60, + ]) + return [1, 1] + } + return 1 + }) + + await cleanupExecutionBase64Cache('exec-1') + + expect(mockRedis.eval).toHaveBeenCalledOnce() + }) + + it('releases indexed budget entries even when cache keys already expired', async () => { + mockGetRedisClient.mockReturnValue(mockRedis) + mockRedis.hgetall.mockResolvedValueOnce({ + 'key:file-1': JSON.stringify({ bytes: 7, userId: 'user-1' }), + }) + mockRedis.eval.mockResolvedValueOnce([1, 0]) + + await cleanupExecutionBase64Cache('exec-1') + + expect(mockRedis.eval).toHaveBeenCalledOnce() + }) + + it('writes execution cache and budget index through one delta-aware script', async () => { + mockGetRedisClient.mockReturnValue(mockRedis) + mockDownloadFile.mockResolvedValueOnce(Buffer.from('hello world!', 'utf8')) + let reservedBytes = 0 + mockRedis.eval.mockImplementation(async (script: string, ...args: unknown[]) => { + if (script.includes('HGET') && script.includes('HSET') && script.includes('SET')) { + const keyCount = Number(args[0]) + const valueBytes = Number(args[keyCount + 5]) + reservedBytes = valueBytes - 10 + return [1, 'ok', reservedBytes, reservedBytes] + } + return 1 + }) + const file: UserFile = { + id: 'file-1', + name: 'delta.txt', + key: 'execution/workspace/workflow/exec-1/delta.txt', + url: '/api/files/serve/execution/workspace/workflow/exec-1/delta.txt?context=execution', + size: 12, + type: 'text/plain', + context: 'execution', + } + + const hydrated = await hydrateUserFilesWithBase64( + { file }, + { + workspaceId: 'workspace', + workflowId: 'workflow', + executionId: 'exec-1', + userId: 'user-1', + maxBytes: 20, + } + ) + + expect(hydrated.file.base64).toBe(Buffer.from('hello world!').toString('base64')) + expect(reservedBytes).toBe(Buffer.from('hello world!').toString('base64').length - 10) + expect(mockRedis.eval).toHaveBeenCalledWith( + expect.stringContaining('HGET'), + 4, + 'user-file:base64:exec:exec-1:key:execution/workspace/workflow/exec-1/delta.txt', + 'user-file:base64-budget:exec:exec-1', + 'execution:redis-budget:execution:exec-1', + 'execution:redis-budget:user:user-1', + Buffer.from('hello world!').toString('base64'), + 60 * 60, + 'key:execution/workspace/workflow/exec-1/delta.txt', + JSON.stringify({ + bytes: Buffer.from('hello world!').toString('base64').length, + userId: 'user-1', + }), + Buffer.from('hello world!').toString('base64').length, + 64 * 1024 * 1024, + 256 * 1024 * 1024, + 60 * 60 + ) + expect(mockRedis.hget).not.toHaveBeenCalled() + expect(mockRedis.set).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/uploads/utils/user-file-base64.server.ts b/apps/sim/lib/uploads/utils/user-file-base64.server.ts index 3aa2f219eb1..8d5e7b048d1 100644 --- a/apps/sim/lib/uploads/utils/user-file-base64.server.ts +++ b/apps/sim/lib/uploads/utils/user-file-base64.server.ts @@ -1,16 +1,136 @@ import type { Logger } from '@sim/logger' import { createLogger } from '@sim/logger' import { getRedisClient } from '@/lib/core/config/redis' -import { getMaxExecutionTimeout } from '@/lib/core/execution-limits' import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' -import { bufferToBase64 } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromStorage, downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' +import { LARGE_VALUE_THRESHOLD_BYTES } from '@/lib/execution/payloads/large-value-ref' +import { + assertUserFileContentAccess, + readUserFileContent, +} from '@/lib/execution/payloads/materialization.server' +import { + type ExecutionRedisBudgetReservation, + getExecutionRedisBudgetKeys, + getExecutionRedisBudgetLimits, +} from '@/lib/execution/redis-budget.server' +import { + ExecutionResourceLimitError, + isExecutionResourceLimitError, +} from '@/lib/execution/resource-errors' import type { UserFile } from '@/executor/types' -const DEFAULT_MAX_BASE64_BYTES = 10 * 1024 * 1024 -const DEFAULT_TIMEOUT_MS = getMaxExecutionTimeout() +const INLINE_BASE64_JSON_OVERHEAD_BYTES = 512 * 1024 +const DEFAULT_MAX_BASE64_BYTES = Math.floor( + (LARGE_VALUE_THRESHOLD_BYTES - INLINE_BASE64_JSON_OVERHEAD_BYTES) * 0.75 +) const DEFAULT_CACHE_TTL_SECONDS = 300 const REDIS_KEY_PREFIX = 'user-file:base64:' +const REDIS_BUDGET_KEY_PREFIX = 'user-file:base64-budget:' +const CLEANUP_BASE64_CACHE_ENTRY_SCRIPT = ` +local file_key = ARGV[1] +local expected_entry = ARGV[2] +local bytes = tonumber(ARGV[3]) +local budget_ttl_seconds = tonumber(ARGV[4]) +local current_entry = redis.call('HGET', KEYS[1], file_key) +if not current_entry or current_entry ~= expected_entry then + return {0, 0} +end +local deleted = redis.call('DEL', KEYS[2]) +redis.call('HDEL', KEYS[1], file_key) +if bytes and bytes > 0 then + local execution_next = redis.call('DECRBY', KEYS[3], bytes) + if execution_next <= 0 then + redis.call('DEL', KEYS[3]) + else + redis.call('EXPIRE', KEYS[3], budget_ttl_seconds) + end + if #KEYS >= 4 then + local user_next = redis.call('DECRBY', KEYS[4], bytes) + if user_next <= 0 then + redis.call('DEL', KEYS[4]) + else + redis.call('EXPIRE', KEYS[4], budget_ttl_seconds) + end + end +end +if redis.call('HLEN', KEYS[1]) == 0 then + redis.call('DEL', KEYS[1]) +end +return {1, deleted} +` +const SET_BASE64_CACHE_SCRIPT = ` +local value = ARGV[1] +local cache_ttl_seconds = tonumber(ARGV[2]) +local file_key = ARGV[3] +local next_entry = ARGV[4] +local next_bytes = tonumber(ARGV[5]) +local execution_limit = tonumber(ARGV[6]) +local user_limit = tonumber(ARGV[7]) +local budget_ttl_seconds = tonumber(ARGV[8]) +local previous_entry = redis.call('HGET', KEYS[2], file_key) +local previous_bytes = 0 +if previous_entry then + local parsed_previous_bytes = string.match(previous_entry, '"bytes"%s*:%s*(%d+)') + if parsed_previous_bytes then + previous_bytes = tonumber(parsed_previous_bytes) + end +end +local execution_current_raw = redis.call('GET', KEYS[3]) +local execution_current = tonumber(execution_current_raw or '0') +local execution_delta = next_bytes - previous_bytes +if not execution_current_raw then + execution_delta = next_bytes +end +if execution_delta > 0 and execution_limit > 0 and execution_current + execution_delta > execution_limit then + return {0, 'execution_redis_bytes', execution_current} +end +local user_delta = 0 +local user_current = 0 +local user_current_raw = nil +if #KEYS >= 4 then + user_current_raw = redis.call('GET', KEYS[4]) + user_current = tonumber(user_current_raw or '0') + user_delta = next_bytes - previous_bytes + if not user_current_raw then + user_delta = next_bytes + end + if user_delta > 0 and user_limit > 0 and user_current + user_delta > user_limit then + return {0, 'user_redis_bytes', user_current} + end +end +if execution_delta > 0 then + redis.call('INCRBY', KEYS[3], execution_delta) +elseif execution_delta < 0 and execution_current_raw then + local execution_next = redis.call('DECRBY', KEYS[3], -execution_delta) + if execution_next <= 0 then + redis.call('DEL', KEYS[3]) + end +end +if redis.call('EXISTS', KEYS[3]) == 1 then + redis.call('EXPIRE', KEYS[3], budget_ttl_seconds) +end +if #KEYS >= 4 then + if user_delta > 0 then + redis.call('INCRBY', KEYS[4], user_delta) + elseif user_delta < 0 and user_current_raw then + local user_next = redis.call('DECRBY', KEYS[4], -user_delta) + if user_next <= 0 then + redis.call('DEL', KEYS[4]) + end + end + if redis.call('EXISTS', KEYS[4]) == 1 then + redis.call('EXPIRE', KEYS[4], budget_ttl_seconds) + end +end +redis.call('SET', KEYS[1], value, 'EX', cache_ttl_seconds) +redis.call('HSET', KEYS[2], file_key, next_entry) +redis.call('EXPIRE', KEYS[2], cache_ttl_seconds) +return {1, 'ok', execution_delta, user_delta} +` + +interface Base64BudgetEntry { + bytes: number + userId?: string +} interface Base64Cache { get(file: UserFile): Promise @@ -25,7 +145,12 @@ interface HydrationState { export interface Base64HydrationOptions { requestId?: string + workspaceId?: string + workflowId?: string executionId?: string + largeValueExecutionIds?: string[] + allowLargeValueWorkflowScope?: boolean + userId?: string logger?: Logger maxBytes?: number allowUnknownSize?: boolean @@ -78,10 +203,62 @@ function createBase64Cache(options: Base64HydrationOptions, logger: Logger): Bas } }, async set(file: UserFile, value: string, ttlSeconds: number) { + const key = getFullCacheKey(executionId, file) + const valueBytes = Buffer.byteLength(value, 'utf8') try { - const key = getFullCacheKey(executionId, file) - await redis.set(key, value, 'EX', ttlSeconds) + if (!executionId) { + await redis.set(key, value, 'EX', ttlSeconds) + return + } + + const limits = getExecutionRedisBudgetLimits() + if (valueBytes > limits.maxSingleWriteBytes) { + throw new ExecutionResourceLimitError({ + resource: 'redis_key_bytes', + attemptedBytes: valueBytes, + limitBytes: limits.maxSingleWriteBytes, + }) + } + const cacheTtlSeconds = Math.max(ttlSeconds, limits.ttlSeconds) + const budgetReservation: ExecutionRedisBudgetReservation = { + executionId, + userId: options.userId, + category: 'base64_cache', + operation: 'set_base64_cache', + bytes: valueBytes, + logger, + } + const budgetKeys = getExecutionRedisBudgetKeys(budgetReservation) + const result = (await redis.eval( + SET_BASE64_CACHE_SCRIPT, + 2 + budgetKeys.length, + key, + getBudgetIndexKey(executionId), + ...budgetKeys, + value, + cacheTtlSeconds, + getFileCacheKey(file), + serializeBudgetEntry({ bytes: valueBytes, userId: options.userId }), + valueBytes, + limits.maxExecutionBytes, + limits.maxUserBytes, + limits.ttlSeconds + )) as [number, string, number | string | null] + const [allowed, resource, current] = result + if (allowed !== 1) { + throw new ExecutionResourceLimitError({ + resource: + resource === 'user_redis_bytes' ? 'user_redis_bytes' : 'execution_redis_bytes', + attemptedBytes: valueBytes, + currentBytes: Number(current ?? 0), + limitBytes: + resource === 'user_redis_bytes' ? limits.maxUserBytes : limits.maxExecutionBytes, + }) + } } catch (error) { + if (isExecutionResourceLimitError(error)) { + throw error + } logger.warn(`[${options.requestId}] Redis set failed, skipping cache`, error) } }, @@ -118,18 +295,87 @@ function getFullCacheKey(executionId: string | undefined, file: UserFile): strin return `${REDIS_KEY_PREFIX}${fileKey}` } +function getBudgetIndexKey(executionId: string): string { + return `${REDIS_BUDGET_KEY_PREFIX}exec:${executionId}` +} + +function serializeBudgetEntry(entry: Base64BudgetEntry): string { + return JSON.stringify(entry) +} + +function parseBudgetEntry(value: unknown): Base64BudgetEntry | null { + if (typeof value !== 'string') { + return null + } + try { + const parsed = JSON.parse(value) as Partial + if (typeof parsed.bytes !== 'number' || !Number.isFinite(parsed.bytes) || parsed.bytes <= 0) { + return null + } + return { + bytes: parsed.bytes, + userId: typeof parsed.userId === 'string' ? parsed.userId : undefined, + } + } catch { + return null + } +} + +async function cleanupBudgetEntry( + redis: NonNullable>, + executionId: string, + fileKey: string, + rawEntry: string, + entry: Base64BudgetEntry +): Promise<{ claimed: boolean; deletedCount: number }> { + const limits = getExecutionRedisBudgetLimits() + const budgetReservation: ExecutionRedisBudgetReservation = { + executionId, + userId: entry.userId, + category: 'base64_cache', + operation: 'cleanup_base64_cache', + bytes: entry.bytes, + } + const budgetKeys = getExecutionRedisBudgetKeys(budgetReservation) + const result = (await redis.eval( + CLEANUP_BASE64_CACHE_ENTRY_SCRIPT, + 2 + budgetKeys.length, + getBudgetIndexKey(executionId), + `${REDIS_KEY_PREFIX}exec:${executionId}:${fileKey}`, + ...budgetKeys, + fileKey, + rawEntry, + entry.bytes, + limits.ttlSeconds + )) as [number, number] + return { claimed: Number(result[0]) === 1, deletedCount: Number(result[1] ?? 0) } +} + +function stripBase64(file: UserFile): UserFile { + const { base64: _base64, ...rest } = file + return rest +} + async function resolveBase64( file: UserFile, options: Base64HydrationOptions, logger: Logger ): Promise { + const requestedMaxBytes = options.maxBytes ?? DEFAULT_MAX_BASE64_BYTES + const maxBytes = Math.min(requestedMaxBytes, DEFAULT_MAX_BASE64_BYTES) + if (file.base64) { + const base64Bytes = Buffer.byteLength(file.base64, 'base64') + if (base64Bytes > maxBytes) { + logger.warn( + `[${options.requestId}] Skipping existing base64 for ${file.name} (decoded ${base64Bytes} exceeds ${maxBytes})` + ) + return null + } return file.base64 } - const maxBytes = options.maxBytes ?? DEFAULT_MAX_BASE64_BYTES const allowUnknownSize = options.allowUnknownSize ?? false - const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS const hasStableStorageKey = Boolean(file.key) if (Number.isFinite(file.size) && file.size > maxBytes) { @@ -148,40 +394,24 @@ async function resolveBase64( return null } - let buffer: Buffer | null = null const requestId = options.requestId ?? 'unknown' - - if (file.key) { - try { - buffer = await downloadFileFromStorage(file, requestId, logger) - } catch (error) { - logger.warn( - `[${requestId}] Failed to download ${file.name} from storage, trying URL fallback`, - error - ) - } - } - - if (!buffer && file.url) { - try { - buffer = await downloadFileFromUrl(file.url, timeoutMs) - } catch (error) { - logger.warn(`[${requestId}] Failed to download ${file.name} from URL`, error) - } - } - - if (!buffer) { - return null - } - - if (buffer.length > maxBytes) { - logger.warn( - `[${options.requestId}] Skipping base64 for ${file.name} (downloaded ${buffer.length} exceeds ${maxBytes})` - ) + try { + return await readUserFileContent(file, { + requestId, + workspaceId: options.workspaceId, + workflowId: options.workflowId, + executionId: options.executionId, + largeValueExecutionIds: options.largeValueExecutionIds, + allowLargeValueWorkflowScope: options.allowLargeValueWorkflowScope, + userId: options.userId, + encoding: 'base64', + maxBytes, + maxSourceBytes: maxBytes, + }) + } catch (error) { + logger.warn(`[${requestId}] Failed to hydrate base64 for ${file.name}`, error) return null } - - return bufferToBase64(buffer) } async function hydrateUserFile( @@ -190,14 +420,39 @@ async function hydrateUserFile( state: HydrationState, logger: Logger ): Promise { + if (!file.base64) { + try { + await assertUserFileContentAccess(file, { + requestId: options.requestId, + workspaceId: options.workspaceId, + workflowId: options.workflowId, + executionId: options.executionId, + largeValueExecutionIds: options.largeValueExecutionIds, + allowLargeValueWorkflowScope: options.allowLargeValueWorkflowScope, + userId: options.userId, + logger, + }) + } catch (error) { + logger.warn(`[${options.requestId ?? 'unknown'}] Skipping unauthorized file base64`, error) + return stripBase64(file) + } + } + const cached = await state.cache.get(file) if (cached) { + const maxBytes = Math.min( + options.maxBytes ?? DEFAULT_MAX_BASE64_BYTES, + DEFAULT_MAX_BASE64_BYTES + ) + if (Buffer.byteLength(cached, 'base64') > maxBytes) { + return stripBase64(file) + } return { ...file, base64: cached } } const base64 = await resolveBase64(file, options, logger) if (!base64) { - return file + return stripBase64(file) } await state.cache.set(file, base64, state.cacheTtlSeconds) @@ -253,6 +508,18 @@ export async function hydrateUserFilesWithBase64( return (await hydrateValue(value, options, state, logger)) as T } +/** + * Hydrates a single UserFile object when a resolver explicitly asks for base64. + */ +export async function hydrateUserFileWithBase64( + file: UserFile, + options: Base64HydrationOptions +): Promise { + const logger = getHydrationLogger(options) + const state = createHydrationState(options, logger) + return hydrateUserFile(file, options, state, logger) +} + function isPlainObject(value: unknown): value is Record { if (!value || typeof value !== 'object') { return false @@ -294,22 +561,25 @@ export async function cleanupExecutionBase64Cache(executionId: string): Promise< return } - const pattern = `${REDIS_KEY_PREFIX}exec:${executionId}:*` const logger = createLogger('UserFileBase64') try { - let cursor = '0' + const budgetEntries = await redis.hgetall(getBudgetIndexKey(executionId)) let deletedCount = 0 - - do { - const [nextCursor, keys] = await redis.scan(cursor, 'MATCH', pattern, 'COUNT', 100) - cursor = nextCursor - - if (keys.length > 0) { - await redis.del(...keys) - deletedCount += keys.length + for (const [fileKey, rawEntry] of Object.entries(budgetEntries ?? {})) { + const budgetEntry = parseBudgetEntry(rawEntry) + if (!budgetEntry) continue + const cleanupResult = await cleanupBudgetEntry( + redis, + executionId, + fileKey, + rawEntry, + budgetEntry + ) + if (cleanupResult.claimed) { + deletedCount += cleanupResult.deletedCount } - } while (cursor !== '0') + } if (deletedCount > 0) { logger.info(`Cleaned up ${deletedCount} base64 cache entries for execution ${executionId}`) diff --git a/apps/sim/lib/workflows/executor/execution-core.ts b/apps/sim/lib/workflows/executor/execution-core.ts index 22b58c5e707..c099ce3151b 100644 --- a/apps/sim/lib/workflows/executor/execution-core.ts +++ b/apps/sim/lib/workflows/executor/execution-core.ts @@ -10,6 +10,7 @@ import { z } from 'zod' import { isPlainRecord } from '@/lib/core/utils/records' import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils' import { clearExecutionCancellation } from '@/lib/execution/cancellation' +import { warmLargeValueRefs } from '@/lib/execution/payloads/hydration' import type { LoggingSession } from '@/lib/logs/execution/logging-session' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' import { @@ -552,10 +553,20 @@ export async function executeWorkflowCore( return persistencePromise } + const largeValueExecutionIds = Array.from( + new Set([executionId, ...(metadata.largeValueExecutionIds ?? [])].filter(Boolean)) + ) + const allowLargeValueWorkflowScope = + metadata.allowLargeValueWorkflowScope === true || + metadata.resumeFromSnapshot === true || + Boolean(runFromBlock?.sourceSnapshot) + const contextExtensions: ContextExtensions = { stream: !!onStream, selectedOutputs, executionId, + largeValueExecutionIds, + allowLargeValueWorkflowScope, workspaceId: providedWorkspaceId, userId, isDeployedContext: !metadata.isClientSession, @@ -582,6 +593,27 @@ export async function executeWorkflowCore( callChain: metadata.callChain, } + if (snapshot.state) { + await warmLargeValueRefs(snapshot.state, { + workspaceId: providedWorkspaceId, + workflowId, + executionId, + largeValueExecutionIds, + allowLargeValueWorkflowScope, + userId, + }) + } + if (runFromBlock?.sourceSnapshot) { + await warmLargeValueRefs(runFromBlock.sourceSnapshot, { + workspaceId: providedWorkspaceId, + workflowId, + executionId, + largeValueExecutionIds, + allowLargeValueWorkflowScope, + userId, + }) + } + for (const variable of Object.values(workflowVariables)) { if ( isPlainRecord(variable) && diff --git a/apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts b/apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts index e4e74a0f982..b41764a0ebd 100644 --- a/apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts +++ b/apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts @@ -13,6 +13,7 @@ import { resetExecutionStreamBuffer, type TerminalExecutionStreamStatus, } from '@/lib/execution/event-buffer' +import { compactBlockLogs, compactExecutionPayload } from '@/lib/execution/payloads/serializer' import { preprocessExecution } from '@/lib/execution/preprocessing' import { LoggingSession } from '@/lib/logs/execution/logging-session' import { executeWorkflowCore } from '@/lib/workflows/executor/execution-core' @@ -25,6 +26,7 @@ import type { SerializableExecutionState, } from '@/executor/execution/types' import type { + BlockLog, ExecutionResult, PauseKind, PausePoint, @@ -980,7 +982,12 @@ export class PauseResumeManager { throw new Error(RUN_BUFFER_UNAVAILABLE_ERROR) } - const eventWriter = createExecutionEventWriter(resumeExecutionId) + const eventWriter = createExecutionEventWriter(resumeExecutionId, { + workspaceId: metadata.workspaceId, + workflowId, + userId: metadata.userId, + preserveUserFileBase64: true, + }) const metaInitialized = await initializeExecutionStreamMeta(resumeExecutionId, { userId: metadata.userId, workflowId, @@ -1197,6 +1204,23 @@ export class PauseResumeManager { } } + const compactResultLogs = await compactBlockLogs(result.logs, { + workspaceId: baseSnapshot.metadata.workspaceId, + workflowId, + executionId: resumeExecutionId, + userId: metadata.userId, + requireDurable: true, + }) + const compactResultOutput = await compactExecutionPayload(result.output, { + workspaceId: baseSnapshot.metadata.workspaceId, + workflowId, + executionId: resumeExecutionId, + userId: metadata.userId, + preserveUserFileBase64: true, + preserveRoot: true, + requireDurable: true, + }) + if ( result.status === 'cancelled' && timeoutController?.isTimedOut() && @@ -1219,7 +1243,7 @@ export class PauseResumeManager { data: { error: timeoutErrorMessage, duration: result.metadata?.duration || 0, - finalBlockLogs: result.logs, + finalBlockLogs: compactResultLogs, }, }, 'error' @@ -1234,7 +1258,7 @@ export class PauseResumeManager { workflowId, data: { duration: result.metadata?.duration || 0, - finalBlockLogs: result.logs, + finalBlockLogs: compactResultLogs, }, }, 'cancelled' @@ -1248,11 +1272,11 @@ export class PauseResumeManager { executionId: resumeExecutionId, workflowId, data: { - output: result.output, + output: compactResultOutput, duration: result.metadata?.duration || 0, startTime: result.metadata?.startTime || new Date().toISOString(), endTime: result.metadata?.endTime || new Date().toISOString(), - finalBlockLogs: result.logs, + finalBlockLogs: compactResultLogs, }, }, 'complete' @@ -1267,11 +1291,11 @@ export class PauseResumeManager { workflowId, data: { success: result.success, - output: result.output, + output: compactResultOutput, duration: result.metadata?.duration || 0, startTime: result.metadata?.startTime || new Date().toISOString(), endTime: result.metadata?.endTime || new Date().toISOString(), - finalBlockLogs: result.logs, + finalBlockLogs: compactResultLogs, }, }, 'complete' @@ -1280,6 +1304,23 @@ export class PauseResumeManager { } catch (execError) { executionError = execError const execErrorResult = hasExecutionResult(execError) ? execError.executionResult : undefined + let compactErrorLogs: BlockLog[] | undefined + try { + compactErrorLogs = execErrorResult?.logs + ? await compactBlockLogs(execErrorResult.logs, { + workspaceId: baseSnapshot.metadata.workspaceId, + workflowId, + executionId: resumeExecutionId, + userId: metadata.userId, + requireDurable: true, + }) + : undefined + } catch (compactionError) { + logger.warn('Failed to compact resume error logs, omitting oversized error details', { + resumeExecutionId, + error: toError(compactionError).message, + }) + } finalMetaStatus = 'error' await writeBufferedEvent( { @@ -1290,7 +1331,7 @@ export class PauseResumeManager { data: { error: toError(execError).message, duration: 0, - finalBlockLogs: execErrorResult?.logs, + finalBlockLogs: compactErrorLogs, }, }, 'error' diff --git a/apps/sim/lib/workflows/persistence/utils.test.ts b/apps/sim/lib/workflows/persistence/utils.test.ts index 82997c4f518..e6b9dbb086a 100644 --- a/apps/sim/lib/workflows/persistence/utils.test.ts +++ b/apps/sim/lib/workflows/persistence/utils.test.ts @@ -179,6 +179,7 @@ const mockBlocksFromDb = [ name: 'Parallel Container', position: { x: 600, y: 50 }, height: 250, + count: 3, data: { width: 500, height: 300, parallelType: 'count', count: 3 }, }), mockWorkflowId @@ -225,7 +226,10 @@ const mockSubflowsFromDb = [ config: { id: 'parallel-1', nodes: ['block-3'], + count: 5, distribution: ['item1', 'item2'], + parallelType: 'count', + batchSize: 1, }, }, ] @@ -260,7 +264,8 @@ const mockWorkflowState = createWorkflowState({ name: 'Parallel Container', position: { x: 600, y: 50 }, height: 250, - data: { width: 500, height: 300, parallelType: 'count', count: 3 }, + count: 3, + data: { width: 500, height: 300, parallelType: 'count', count: 3, batchSize: 1 }, }), 'block-3': createApiBlock({ id: 'block-3', @@ -292,6 +297,8 @@ const mockWorkflowState = createWorkflowState({ id: 'parallel-1', nodes: ['block-3'], distribution: ['item1', 'item2'], + parallelType: 'count', + batchSize: 1, }, }, }) @@ -418,8 +425,16 @@ describe('Database Helpers', () => { count: 5, distribution: ['item1', 'item2'], parallelType: 'count', + batchSize: 1, enabled: true, }) + expect(result?.blocks['parallel-1'].data).toEqual( + expect.objectContaining({ + count: 5, + parallelType: 'count', + batchSize: 1, + }) + ) }) it('should return null when no blocks are found', async () => { @@ -709,6 +724,20 @@ describe('Database Helpers', () => { workflowId: mockWorkflowId, type: 'loop', }) + expect(capturedSubflowInserts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 'parallel-1', + workflowId: mockWorkflowId, + type: 'parallel', + config: expect.objectContaining({ + count: 3, + parallelType: 'count', + batchSize: 1, + }), + }), + ]) + ) }) it('should regenerate missing loop and parallel definitions from block data', async () => { @@ -748,7 +777,11 @@ describe('Database Helpers', () => { expect(capturedSubflowInserts).toEqual( expect.arrayContaining([ expect.objectContaining({ id: 'loop-1', type: 'loop' }), - expect.objectContaining({ id: 'parallel-1', type: 'parallel' }), + expect.objectContaining({ + id: 'parallel-1', + type: 'parallel', + config: expect.objectContaining({ batchSize: 1 }), + }), ]) ) }) diff --git a/apps/sim/lib/workflows/search-replace/replacements.test.ts b/apps/sim/lib/workflows/search-replace/replacements.test.ts index 8dd90198605..3ea5963acaa 100644 --- a/apps/sim/lib/workflows/search-replace/replacements.test.ts +++ b/apps/sim/lib/workflows/search-replace/replacements.test.ts @@ -1201,6 +1201,13 @@ describe('buildWorkflowSearchReplacePlan', () => { expect(countPlan.conflicts).toEqual([]) expect(countPlan.subflowUpdates).toEqual([ + expect.objectContaining({ + blockId: 'parallel-1', + blockType: 'parallel', + fieldId: WORKFLOW_SEARCH_SUBFLOW_FIELD_IDS.batchSize, + previousValue: '20', + nextValue: 3, + }), expect.objectContaining({ blockId: 'parallel-1', blockType: 'parallel', @@ -1569,8 +1576,8 @@ describe('buildWorkflowSearchReplacePlan', () => { expect(plan.subflowUpdates).toEqual([]) expect(plan.conflicts).toEqual([ { - matchId: matches[0].id, - reason: 'Subflow iteration count must be between 1 and 20', + matchId: 'subflow-text:parallel-1:subflowBatchSize:0:0', + reason: 'Parallel batch size must be between 1 and 20', }, ]) }) diff --git a/apps/sim/lib/workflows/search-replace/subflow-fields.ts b/apps/sim/lib/workflows/search-replace/subflow-fields.ts index c87b982efb2..6f46d9039e8 100644 --- a/apps/sim/lib/workflows/search-replace/subflow-fields.ts +++ b/apps/sim/lib/workflows/search-replace/subflow-fields.ts @@ -5,6 +5,7 @@ export const WORKFLOW_SEARCH_SUBFLOW_FIELD_IDS = { iterations: 'subflowIterations', items: 'subflowItems', condition: 'subflowCondition', + batchSize: 'subflowBatchSize', } as const export type WorkflowSearchSubflowFieldId = @@ -18,6 +19,7 @@ interface WorkflowSearchSubflowBlock { loopType?: string parallelType?: string count?: unknown + batchSize?: unknown collection?: unknown whileCondition?: unknown doWhileCondition?: unknown @@ -113,6 +115,14 @@ export function getWorkflowSearchSubflowFields( editable: true, valueKind: parallelType === 'count' ? 'number' : 'text', }, + { + id: WORKFLOW_SEARCH_SUBFLOW_FIELD_IDS.batchSize, + title: 'Parallel Batch Size', + type: 'short-input', + value: String(block.data?.batchSize ?? 20), + editable: true, + valueKind: 'number', + }, ] } @@ -146,7 +156,10 @@ export function parseWorkflowSearchSubflowReplacement({ }): | { success: true; value: WorkflowSearchSubflowEditableValue } | { success: false; reason: string } { - if (fieldId !== WORKFLOW_SEARCH_SUBFLOW_FIELD_IDS.iterations) { + if ( + fieldId !== WORKFLOW_SEARCH_SUBFLOW_FIELD_IDS.iterations && + fieldId !== WORKFLOW_SEARCH_SUBFLOW_FIELD_IDS.batchSize + ) { return { success: true, value: replacement } } @@ -156,11 +169,17 @@ export function parseWorkflowSearchSubflowReplacement({ } const count = Number.parseInt(trimmed, 10) - const max = blockType === 'parallel' ? 20 : 1000 - if (count < 1 || count > max) { + const maxBatchSize = 20 + if ( + count < 1 || + (fieldId === WORKFLOW_SEARCH_SUBFLOW_FIELD_IDS.batchSize && count > maxBatchSize) + ) { return { success: false, - reason: `Subflow iteration count must be between 1 and ${max}`, + reason: + fieldId === WORKFLOW_SEARCH_SUBFLOW_FIELD_IDS.batchSize + ? `Parallel batch size must be between 1 and ${maxBatchSize}` + : 'Subflow iteration count must be greater than 0', } } diff --git a/apps/sim/lib/workflows/streaming/streaming.ts b/apps/sim/lib/workflows/streaming/streaming.ts index d4f881e78c6..3336f17a9c2 100644 --- a/apps/sim/lib/workflows/streaming/streaming.ts +++ b/apps/sim/lib/workflows/streaming/streaming.ts @@ -6,6 +6,7 @@ import { traverseObjectPath, } from '@/lib/core/utils/response-format' import { encodeSSE } from '@/lib/core/utils/sse' +import { compactExecutionPayload } from '@/lib/execution/payloads/serializer' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' import { processStreamingBlockLogs } from '@/lib/tokenization' import { @@ -45,6 +46,11 @@ export interface StreamingResponseOptions { requestId: string streamConfig: StreamingConfig executionId?: string + largeValueExecutionIds?: string[] + allowLargeValueWorkflowScope?: boolean + workspaceId?: string + workflowId?: string + userId?: string executeFn: StreamingExecutorFn } @@ -78,8 +84,18 @@ async function buildMinimalResult( completedBlockIds: Set, requestId: string, includeFileBase64: boolean, - base64MaxBytes: number | undefined + base64MaxBytes: number | undefined, + executionId?: string, + context: Pick = {} ): Promise<{ success: boolean; error?: string; output: Record }> { + const durableContext = { + workspaceId: context.workspaceId, + workflowId: context.workflowId, + executionId, + userId: context.userId, + requireDurable: Boolean(context.workspaceId && context.workflowId && executionId), + } + const minimalResult = { success: result.success, error: result.error, @@ -88,12 +104,20 @@ async function buildMinimalResult( if (result.status === 'paused') { minimalResult.output = result.output || {} - return minimalResult + return compactExecutionPayload(minimalResult, { + ...durableContext, + preserveUserFileBase64: includeFileBase64, + preserveRoot: true, + }) } if (!selectedOutputs?.length) { minimalResult.output = result.output || {} - return minimalResult + return compactExecutionPayload(minimalResult, { + ...durableContext, + preserveUserFileBase64: includeFileBase64, + preserveRoot: true, + }) } if (!result.output || !result.logs) { @@ -138,7 +162,11 @@ async function buildMinimalResult( ;(minimalResult.output[blockId] as Record)[path] = value } - return minimalResult + return compactExecutionPayload(minimalResult, { + ...durableContext, + preserveUserFileBase64: includeFileBase64, + preserveRoot: true, + }) } function updateLogsWithStreamedContent( @@ -191,6 +219,13 @@ export async function createStreamingResponse( options: StreamingResponseOptions ): Promise { const { requestId, streamConfig, executionId, executeFn } = options + const durableContext = { + workspaceId: options.workspaceId, + workflowId: options.workflowId, + executionId, + userId: options.userId, + requireDurable: Boolean(options.workspaceId && options.workflowId && executionId), + } const timeoutController = createTimeoutAbortController(streamConfig.timeoutMs) return new ReadableStream({ @@ -281,14 +316,23 @@ export async function createStreamingResponse( const hydratedOutput = includeFileBase64 ? await hydrateUserFilesWithBase64(outputValue, { requestId, + workspaceId: options.workspaceId, + workflowId: options.workflowId, executionId, + largeValueExecutionIds: options.largeValueExecutionIds, + allowLargeValueWorkflowScope: options.allowLargeValueWorkflowScope, + userId: options.userId, maxBytes: base64MaxBytes, }) : outputValue + const compactHydratedOutput = await compactExecutionPayload(hydratedOutput, { + ...durableContext, + preserveUserFileBase64: includeFileBase64, + }) const formattedOutput = - typeof hydratedOutput === 'string' - ? hydratedOutput - : JSON.stringify(hydratedOutput, null, 2) + typeof compactHydratedOutput === 'string' + ? compactHydratedOutput + : JSON.stringify(compactHydratedOutput, null, 2) sendChunk(blockId, formattedOutput) } } @@ -336,7 +380,13 @@ export async function createStreamingResponse( state.completedBlockIds, requestId, streamConfig.includeFileBase64 ?? true, - streamConfig.base64MaxBytes + streamConfig.base64MaxBytes, + executionId, + { + workspaceId: options.workspaceId, + workflowId: options.workflowId, + userId: options.userId, + } ) controller.enqueue( diff --git a/apps/sim/lib/workflows/utils.ts b/apps/sim/lib/workflows/utils.ts index 318d6249d6a..30afa6d81d3 100644 --- a/apps/sim/lib/workflows/utils.ts +++ b/apps/sim/lib/workflows/utils.ts @@ -6,6 +6,7 @@ import { authorizeWorkflowByWorkspacePermission } from '@sim/workflow-authz' import { and, asc, eq, inArray, isNull, max, min, sql } from 'drizzle-orm' import { NextResponse } from 'next/server' import { getSession } from '@/lib/auth' +import { materializeLargeValueRefsSync } from '@/lib/execution/payloads/cache' import { getNextWorkflowColor } from '@/lib/workflows/colors' import { buildDefaultWorkflowArtifacts } from '@/lib/workflows/defaults' import { saveWorkflowToNormalizedTables } from '@/lib/workflows/persistence/utils' @@ -319,13 +320,14 @@ export const createHttpResponseFromBlock = ( executionResult: Pick ): NextResponse => { const { data = {}, status = 200, headers = {} } = executionResult.output + const responseData = materializeLargeValueRefsSync(data) const responseHeaders = new Headers({ 'Content-Type': 'application/json', ...headers, }) - return NextResponse.json(data, { + return NextResponse.json(responseData, { status: status, headers: responseHeaders, }) diff --git a/apps/sim/proxy.ts b/apps/sim/proxy.ts index 5a2e2796269..ed642956360 100644 --- a/apps/sim/proxy.ts +++ b/apps/sim/proxy.ts @@ -203,6 +203,6 @@ export const config = { '/signup', '/invite/:path*', // Match invitation routes // Catch-all for other pages, excluding static assets and public directories - '/((?!_next/static|_next/image|ingest|favicon.ico|logo/|static/|footer/|social/|enterprise/|favicon/|twitter/|robots.txt|sitemap.xml).*)', + '/((?!api/|api$|_next/static|_next/image|ingest|favicon.ico|logo/|static/|footer/|social/|enterprise/|favicon/|twitter/|robots.txt|sitemap.xml).*)', ], } diff --git a/apps/sim/serializer/types.ts b/apps/sim/serializer/types.ts index 8192014a4ae..8d7bc56e4ed 100644 --- a/apps/sim/serializer/types.ts +++ b/apps/sim/serializer/types.ts @@ -58,4 +58,5 @@ export interface SerializedParallel { distribution?: any[] | Record | string // Items to distribute or expression to evaluate count?: number // Number of parallel executions for count-based parallel parallelType?: 'count' | 'collection' // Explicit parallel type to avoid inference bugs + batchSize?: number // Maximum number of branches to run concurrently per batch } diff --git a/apps/sim/stores/workflows/workflow/store.test.ts b/apps/sim/stores/workflows/workflow/store.test.ts index dc24da784e6..720fee128b8 100644 --- a/apps/sim/stores/workflows/workflow/store.test.ts +++ b/apps/sim/stores/workflows/workflow/store.test.ts @@ -500,7 +500,7 @@ describe('workflow store', () => { expect(state.loops.loop1.forEachItems).toBe('["item1", "item2", "item3"]') }) - it('should clamp loop count between 1 and 1000', () => { + it('should allow loop counts above 1000 and clamp only to at least 1', () => { const { updateLoopCount } = useWorkflowStore.getState() addBlock( @@ -517,7 +517,7 @@ describe('workflow store', () => { updateLoopCount('loop1', 1500) let state = useWorkflowStore.getState() - expect(state.blocks.loop1?.data?.count).toBe(1000) + expect(state.blocks.loop1?.data?.count).toBe(1500) updateLoopCount('loop1', 0) state = useWorkflowStore.getState() @@ -576,7 +576,7 @@ describe('workflow store', () => { expect(parsedDistribution).toHaveLength(3) }) - it('should clamp parallel count between 1 and 20', () => { + it('should allow parallel counts above 1000 and clamp only to at least 1', () => { const { updateParallelCount } = useWorkflowStore.getState() addBlock( @@ -592,13 +592,46 @@ describe('workflow store', () => { updateParallelCount('parallel1', 100) let state = useWorkflowStore.getState() - expect(state.blocks.parallel1?.data?.count).toBe(20) + expect(state.blocks.parallel1?.data?.count).toBe(100) + + updateParallelCount('parallel1', 1001) + state = useWorkflowStore.getState() + expect(state.blocks.parallel1?.data?.count).toBe(1001) updateParallelCount('parallel1', 0) state = useWorkflowStore.getState() expect(state.blocks.parallel1?.data?.count).toBe(1) }) + it('should clamp parallel batch size between 1 and 20', () => { + const { updateParallelBatchSize } = useWorkflowStore.getState() + + addBlock( + 'parallel1', + 'parallel', + 'Test Parallel', + { x: 0, y: 0 }, + { + count: 5, + batchSize: 20, + collection: '', + } + ) + + updateParallelBatchSize('parallel1', 7) + let state = useWorkflowStore.getState() + expect(state.blocks.parallel1?.data?.batchSize).toBe(7) + expect(state.parallels.parallel1.batchSize).toBe(7) + + updateParallelBatchSize('parallel1', 50) + state = useWorkflowStore.getState() + expect(state.blocks.parallel1?.data?.batchSize).toBe(20) + + updateParallelBatchSize('parallel1', 0) + state = useWorkflowStore.getState() + expect(state.blocks.parallel1?.data?.batchSize).toBe(1) + }) + it('should regenerate parallels when updateParallelType is called', () => { const { updateParallelType } = useWorkflowStore.getState() diff --git a/apps/sim/stores/workflows/workflow/store.ts b/apps/sim/stores/workflows/workflow/store.ts index 888bf069bef..e6fd406b80e 100644 --- a/apps/sim/stores/workflows/workflow/store.ts +++ b/apps/sim/stores/workflows/workflow/store.ts @@ -26,6 +26,7 @@ import type { WorkflowStore, } from '@/stores/workflows/workflow/types' import { + clampParallelBatchSize, findAllDescendantNodes, generateLoopBlocks, generateParallelBlocks, @@ -995,7 +996,7 @@ export const useWorkflowStore = create()( ...block, data: { ...block.data, - count: Math.max(1, Math.min(1000, count)), // Clamp between 1-1000 + count: Math.max(1, count), }, }, } @@ -1163,7 +1164,7 @@ export const useWorkflowStore = create()( ...block, data: { ...block.data, - count: Math.max(1, Math.min(20, count)), // Clamp between 1-20 + count: Math.max(1, count), }, }, } @@ -1180,6 +1181,32 @@ export const useWorkflowStore = create()( // Note: Socket.IO handles real-time sync automatically }, + updateParallelBatchSize: (parallelId: string, batchSize: number) => { + const block = get().blocks[parallelId] + if (!block || block.type !== 'parallel') return + + const newBlocks = { + ...get().blocks, + [parallelId]: { + ...block, + data: { + ...block.data, + batchSize: clampParallelBatchSize(batchSize), + }, + }, + } + + const newState = { + blocks: newBlocks, + edges: [...get().edges], + loops: { ...get().loops }, + parallels: generateParallelBlocks(newBlocks), + } + + set(newState) + get().updateLastSaved() + }, + updateParallelCollection: (parallelId: string, collection: string) => { const block = get().blocks[parallelId] if (!block || block.type !== 'parallel') return diff --git a/apps/sim/stores/workflows/workflow/types.ts b/apps/sim/stores/workflows/workflow/types.ts index c209cfd0eef..1f32f318764 100644 --- a/apps/sim/stores/workflows/workflow/types.ts +++ b/apps/sim/stores/workflows/workflow/types.ts @@ -84,6 +84,7 @@ export interface WorkflowActions { setLoopWhileCondition: (loopId: string, condition: string) => void setLoopDoWhileCondition: (loopId: string, condition: string) => void updateParallelCount: (parallelId: string, count: number) => void + updateParallelBatchSize: (parallelId: string, batchSize: number) => void updateParallelCollection: (parallelId: string, collection: string) => void updateParallelType: (parallelId: string, parallelType: 'count' | 'collection') => void generateLoopBlocks: () => Record diff --git a/apps/sim/stores/workflows/workflow/utils.ts b/apps/sim/stores/workflows/workflow/utils.ts index 26c2f642a85..a7077dc0903 100644 --- a/apps/sim/stores/workflows/workflow/utils.ts +++ b/apps/sim/stores/workflows/workflow/utils.ts @@ -6,6 +6,16 @@ import type { Edge } from 'reactflow' import type { BlockState, Loop, Parallel } from '@/stores/workflows/workflow/types' const DEFAULT_LOOP_ITERATIONS = 5 +const DEFAULT_PARALLEL_BATCH_SIZE = 20 +const MAX_PARALLEL_BATCH_SIZE = 20 + +export function clampParallelBatchSize(batchSize: unknown): number { + const parsed = typeof batchSize === 'number' ? batchSize : Number.parseInt(String(batchSize), 10) + if (Number.isNaN(parsed)) { + return DEFAULT_PARALLEL_BATCH_SIZE + } + return Math.max(1, Math.min(MAX_PARALLEL_BATCH_SIZE, parsed)) +} /** * Check if adding an edge would create a cycle in the graph. @@ -111,6 +121,7 @@ export function convertParallelBlockToParallel( validatedParallelType === 'collection' ? parallelBlock.data?.collection || '' : undefined const count = parallelBlock.data?.count || 5 + const batchSize = clampParallelBatchSize(parallelBlock.data?.batchSize) return { id: parallelBlockId, @@ -118,6 +129,7 @@ export function convertParallelBlockToParallel( distribution, count, parallelType: validatedParallelType, + batchSize, enabled: parallelBlock.enabled, } } diff --git a/apps/sim/tools/function/execute.test.ts b/apps/sim/tools/function/execute.test.ts index 73eb21de9e6..b174634e57f 100644 --- a/apps/sim/tools/function/execute.test.ts +++ b/apps/sim/tools/function/execute.test.ts @@ -66,6 +66,7 @@ describe('Function Execute Tool', () => { outputTable: undefined, timeout: 5000, workflowId: undefined, + executionId: undefined, workspaceId: undefined, userId: undefined, }) @@ -101,6 +102,7 @@ describe('Function Execute Tool', () => { outputSandboxPath: undefined, outputTable: undefined, workflowId: undefined, + executionId: undefined, workspaceId: undefined, userId: undefined, }) @@ -128,6 +130,7 @@ describe('Function Execute Tool', () => { outputSandboxPath: undefined, outputTable: undefined, workflowId: undefined, + executionId: undefined, workspaceId: undefined, userId: undefined, }) diff --git a/apps/sim/tools/function/execute.ts b/apps/sim/tools/function/execute.ts index 4d096ce7cf4..6821131b30a 100644 --- a/apps/sim/tools/function/execute.ts +++ b/apps/sim/tools/function/execute.ts @@ -137,6 +137,9 @@ export const functionExecuteTool: ToolConfig _context?: { workflowId?: string + executionId?: string + largeValueExecutionIds?: string[] + allowLargeValueWorkflowScope?: boolean userId?: string workspaceId?: string } diff --git a/apps/sim/tools/grafana/create_alert_rule.ts b/apps/sim/tools/grafana/create_alert_rule.ts index a1f4fea02bc..0c07eea7683 100644 --- a/apps/sim/tools/grafana/create_alert_rule.ts +++ b/apps/sim/tools/grafana/create_alert_rule.ts @@ -2,6 +2,8 @@ import type { GrafanaCreateAlertRuleParams, GrafanaCreateAlertRuleResponse, } from '@/tools/grafana/types' +import { ALERT_RULE_OUTPUT_FIELDS } from '@/tools/grafana/types' +import { mapAlertRule } from '@/tools/grafana/utils' import type { ToolConfig } from '@/tools/types' export const createAlertRuleTool: ToolConfig< @@ -52,9 +54,10 @@ export const createAlertRuleTool: ToolConfig< }, condition: { type: 'string', - required: true, + required: false, visibility: 'user-or-llm', - description: 'The refId of the query or expression to use as the alert condition', + description: + 'The refId of the query or expression to use as the alert condition (required for alerting rules; omit for recording rules)', }, data: { type: 'string', @@ -78,7 +81,7 @@ export const createAlertRuleTool: ToolConfig< type: 'string', required: false, visibility: 'user-only', - description: 'State on execution error (Alerting, OK)', + description: 'State on execution error (Error, Alerting, OK)', }, annotations: { type: 'string', @@ -92,6 +95,48 @@ export const createAlertRuleTool: ToolConfig< visibility: 'user-or-llm', description: 'JSON object of labels', }, + uid: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Optional custom UID for the alert rule', + }, + isPaused: { + type: 'boolean', + required: false, + visibility: 'user-only', + description: 'Whether the rule is paused on creation', + }, + keepFiringFor: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Duration to keep firing after the condition stops (e.g., 5m)', + }, + missingSeriesEvalsToResolve: { + type: 'number', + required: false, + visibility: 'user-only', + description: 'Number of missing series evaluations before resolving', + }, + notificationSettings: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'JSON object of per-rule notification settings (overrides)', + }, + record: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'JSON object configuring this as a recording rule (omit for alerting rules)', + }, + disableProvenance: { + type: 'boolean', + required: false, + visibility: 'user-only', + description: 'Set X-Disable-Provenance header so the rule remains editable in the Grafana UI', + }, }, request: { @@ -105,32 +150,43 @@ export const createAlertRuleTool: ToolConfig< if (params.organizationId) { headers['X-Grafana-Org-Id'] = params.organizationId } + if (params.disableProvenance) { + headers['X-Disable-Provenance'] = 'true' + } return headers }, body: (params) => { - let dataArray: any[] = [] + let dataArray: unknown[] = [] try { dataArray = JSON.parse(params.data) } catch { throw new Error('Invalid JSON for data parameter') } - const body: Record = { + const body: Record = { title: params.title, folderUID: params.folderUid, ruleGroup: params.ruleGroup, - condition: params.condition, data: dataArray, - for: params.forDuration || '5m', - noDataState: params.noDataState || 'NoData', - execErrState: params.execErrState || 'Alerting', + } + if (params.organizationId) body.orgID = Number(params.organizationId) + + if (params.condition) body.condition = params.condition + if (params.uid) body.uid = params.uid + if (params.forDuration) body.for = params.forDuration + if (params.noDataState) body.noDataState = params.noDataState + if (params.execErrState) body.execErrState = params.execErrState + if (params.isPaused !== undefined) body.isPaused = params.isPaused + if (params.keepFiringFor) body.keep_firing_for = params.keepFiringFor + if (params.missingSeriesEvalsToResolve !== undefined) { + body.missingSeriesEvalsToResolve = params.missingSeriesEvalsToResolve } if (params.annotations) { try { body.annotations = JSON.parse(params.annotations) } catch { - body.annotations = {} + throw new Error('Invalid JSON for annotations parameter') } } @@ -138,7 +194,23 @@ export const createAlertRuleTool: ToolConfig< try { body.labels = JSON.parse(params.labels) } catch { - body.labels = {} + throw new Error('Invalid JSON for labels parameter') + } + } + + if (params.notificationSettings) { + try { + body.notification_settings = JSON.parse(params.notificationSettings) + } catch { + throw new Error('Invalid JSON for notificationSettings parameter') + } + } + + if (params.record) { + try { + body.record = JSON.parse(params.record) + } catch { + throw new Error('Invalid JSON for record parameter') } } @@ -148,47 +220,8 @@ export const createAlertRuleTool: ToolConfig< transformResponse: async (response: Response) => { const data = await response.json() - - return { - success: true, - output: { - uid: data.uid, - title: data.title, - condition: data.condition, - data: data.data, - updated: data.updated, - noDataState: data.noDataState, - execErrState: data.execErrState, - for: data.for, - annotations: data.annotations || {}, - labels: data.labels || {}, - isPaused: data.isPaused || false, - folderUID: data.folderUID, - ruleGroup: data.ruleGroup, - orgId: data.orgId, - namespace_uid: data.namespace_uid, - namespace_id: data.namespace_id, - provenance: data.provenance || '', - }, - } + return { success: true, output: mapAlertRule(data) } }, - outputs: { - uid: { - type: 'string', - description: 'The UID of the created alert rule', - }, - title: { - type: 'string', - description: 'Alert rule title', - }, - folderUID: { - type: 'string', - description: 'Parent folder UID', - }, - ruleGroup: { - type: 'string', - description: 'Rule group name', - }, - }, + outputs: ALERT_RULE_OUTPUT_FIELDS, } diff --git a/apps/sim/tools/grafana/create_annotation.ts b/apps/sim/tools/grafana/create_annotation.ts index dc75e3f5881..9b3c8859ed5 100644 --- a/apps/sim/tools/grafana/create_annotation.ts +++ b/apps/sim/tools/grafana/create_annotation.ts @@ -46,9 +46,10 @@ export const createAnnotationTool: ToolConfig< }, dashboardUid: { type: 'string', - required: true, + required: false, visibility: 'user-or-llm', - description: 'UID of the dashboard to add the annotation to (e.g., abc123def)', + description: + 'UID of the dashboard to add the annotation to (e.g., abc123def). Omit to create a global organization annotation.', }, panelId: { type: 'number', @@ -84,11 +85,15 @@ export const createAnnotationTool: ToolConfig< return headers }, body: (params) => { - const body: Record = { + const body: Record = { text: params.text, - time: params.time || Date.now(), } + if (params.time) body.time = params.time + if (params.timeEnd) body.timeEnd = params.timeEnd + if (params.dashboardUid) body.dashboardUID = params.dashboardUid + if (params.panelId) body.panelId = params.panelId + if (params.tags) { body.tags = params.tags .split(',') @@ -96,18 +101,6 @@ export const createAnnotationTool: ToolConfig< .filter((t) => t) } - if (params.dashboardUid) { - body.dashboardUID = params.dashboardUid - } - - if (params.panelId) { - body.panelId = params.panelId - } - - if (params.timeEnd) { - body.timeEnd = params.timeEnd - } - return body }, }, diff --git a/apps/sim/tools/grafana/create_folder.ts b/apps/sim/tools/grafana/create_folder.ts index 3231690d3b5..228a9652590 100644 --- a/apps/sim/tools/grafana/create_folder.ts +++ b/apps/sim/tools/grafana/create_folder.ts @@ -39,6 +39,12 @@ export const createFolderTool: ToolConfig { - const body: Record = { + const body: Record = { title: params.title, } - if (params.uid) { - body.uid = params.uid - } + if (params.uid) body.uid = params.uid + if (params.parentUid) body.parentUid = params.parentUid return body }, @@ -73,80 +78,80 @@ export const createFolderTool: ToolConfig = @@ -53,71 +58,8 @@ export const getAlertRuleTool: ToolConfig { const data = await response.json() - - return { - success: true, - output: { - uid: data.uid, - title: data.title, - condition: data.condition, - data: data.data, - updated: data.updated, - noDataState: data.noDataState, - execErrState: data.execErrState, - for: data.for, - annotations: data.annotations || {}, - labels: data.labels || {}, - isPaused: data.isPaused || false, - folderUID: data.folderUID, - ruleGroup: data.ruleGroup, - orgId: data.orgId, - namespace_uid: data.namespace_uid, - namespace_id: data.namespace_id, - provenance: data.provenance || '', - }, - } + return { success: true, output: mapAlertRule(data) } }, - outputs: { - uid: { - type: 'string', - description: 'Alert rule UID', - }, - title: { - type: 'string', - description: 'Alert rule title', - }, - condition: { - type: 'string', - description: 'Alert condition', - }, - data: { - type: 'json', - description: 'Alert rule query data', - }, - folderUID: { - type: 'string', - description: 'Parent folder UID', - }, - ruleGroup: { - type: 'string', - description: 'Rule group name', - }, - noDataState: { - type: 'string', - description: 'State when no data is returned', - }, - execErrState: { - type: 'string', - description: 'State on execution error', - }, - annotations: { - type: 'json', - description: 'Alert annotations', - }, - labels: { - type: 'json', - description: 'Alert labels', - }, - }, + outputs: ALERT_RULE_OUTPUT_FIELDS, } diff --git a/apps/sim/tools/grafana/get_data_source.ts b/apps/sim/tools/grafana/get_data_source.ts index e70377a5575..1ef7bfaa1bc 100644 --- a/apps/sim/tools/grafana/get_data_source.ts +++ b/apps/sim/tools/grafana/get_data_source.ts @@ -43,12 +43,14 @@ export const getDataSourceTool: ToolConfig< request: { url: (params) => { const baseUrl = params.baseUrl.replace(/\/$/, '') - // Check if it looks like a UID (contains non-numeric characters) or ID - const isUid = /[^0-9]/.test(params.dataSourceId) - if (isUid) { - return `${baseUrl}/api/datasources/uid/${params.dataSourceId}` + const id = params.dataSourceId.trim() + // Numeric DB id route only matches purely-numeric ids up to int64 length; + // anything else is treated as a UID (Grafana UIDs are short slug strings). + const isNumericId = /^\d+$/.test(id) && id.length <= 18 + if (isNumericId) { + return `${baseUrl}/api/datasources/${id}` } - return `${baseUrl}/api/datasources/${params.dataSourceId}` + return `${baseUrl}/api/datasources/uid/${id}` }, method: 'GET', headers: (params) => { @@ -69,57 +71,54 @@ export const getDataSourceTool: ToolConfig< return { success: true, output: { - id: data.id, - uid: data.uid, - orgId: data.orgId, - name: data.name, - type: data.type, - typeName: data.typeName, - typeLogoUrl: data.typeLogoUrl, - access: data.access, - url: data.url, - user: data.user, - database: data.database, - basicAuth: data.basicAuth || false, - isDefault: data.isDefault || false, - jsonData: data.jsonData || {}, - readOnly: data.readOnly || false, + id: (data.id as number) ?? null, + uid: (data.uid as string) ?? null, + orgId: (data.orgId as number) ?? null, + name: (data.name as string) ?? null, + type: (data.type as string) ?? null, + typeLogoUrl: (data.typeLogoUrl as string) ?? null, + access: (data.access as string) ?? null, + url: (data.url as string) ?? null, + user: (data.user as string) ?? null, + database: (data.database as string) ?? null, + basicAuth: (data.basicAuth as boolean) ?? false, + basicAuthUser: (data.basicAuthUser as string) ?? null, + withCredentials: (data.withCredentials as boolean) ?? null, + isDefault: (data.isDefault as boolean) ?? false, + jsonData: (data.jsonData as Record) ?? {}, + secureJsonFields: (data.secureJsonFields as Record) ?? {}, + version: (data.version as number) ?? null, + readOnly: (data.readOnly as boolean) ?? false, }, } }, outputs: { - id: { - type: 'number', - description: 'Data source ID', - }, - uid: { - type: 'string', - description: 'Data source UID', - }, - name: { - type: 'string', - description: 'Data source name', - }, - type: { - type: 'string', - description: 'Data source type', - }, - url: { - type: 'string', - description: 'Data source connection URL', - }, - database: { - type: 'string', - description: 'Database name (if applicable)', - }, - isDefault: { + id: { type: 'number', description: 'Data source ID' }, + uid: { type: 'string', description: 'Data source UID' }, + orgId: { type: 'number', description: 'Organization ID' }, + name: { type: 'string', description: 'Data source name' }, + type: { type: 'string', description: 'Data source type' }, + typeLogoUrl: { type: 'string', description: 'Logo URL for the data source type' }, + access: { type: 'string', description: 'Access mode (proxy or direct)' }, + url: { type: 'string', description: 'Data source connection URL' }, + user: { type: 'string', description: 'Username used to connect' }, + database: { type: 'string', description: 'Database name (if applicable)' }, + basicAuth: { type: 'boolean', description: 'Whether basic auth is enabled' }, + basicAuthUser: { type: 'string', description: 'Basic auth username', optional: true }, + withCredentials: { type: 'boolean', - description: 'Whether this is the default data source', + description: 'Whether to send credentials with cross-origin requests', + optional: true, }, - jsonData: { - type: 'json', - description: 'Additional data source configuration', + isDefault: { type: 'boolean', description: 'Whether this is the default data source' }, + jsonData: { type: 'json', description: 'Additional data source configuration' }, + secureJsonFields: { + type: 'object', + description: 'Map of secure fields that are set (values are not returned)', + optional: true, }, + version: { type: 'number', description: 'Data source version', optional: true }, + readOnly: { type: 'boolean', description: 'Whether the data source is read-only' }, }, } diff --git a/apps/sim/tools/grafana/list_alert_rules.ts b/apps/sim/tools/grafana/list_alert_rules.ts index 8f85c56ab28..f4b2b8b978e 100644 --- a/apps/sim/tools/grafana/list_alert_rules.ts +++ b/apps/sim/tools/grafana/list_alert_rules.ts @@ -1,7 +1,9 @@ -import type { - GrafanaListAlertRulesParams, - GrafanaListAlertRulesResponse, +import { + ALERT_RULE_OUTPUT_FIELDS, + type GrafanaListAlertRulesParams, + type GrafanaListAlertRulesResponse, } from '@/tools/grafana/types' +import { mapAlertRule } from '@/tools/grafana/utils' import type { ToolConfig } from '@/tools/types' export const listAlertRulesTool: ToolConfig< @@ -56,25 +58,7 @@ export const listAlertRulesTool: ToolConfig< success: true, output: { rules: Array.isArray(data) - ? data.map((rule: any) => ({ - uid: rule.uid, - title: rule.title, - condition: rule.condition, - data: rule.data, - updated: rule.updated, - noDataState: rule.noDataState, - execErrState: rule.execErrState, - for: rule.for, - annotations: rule.annotations || {}, - labels: rule.labels || {}, - isPaused: rule.isPaused || false, - folderUID: rule.folderUID, - ruleGroup: rule.ruleGroup, - orgId: rule.orgId, - namespace_uid: rule.namespace_uid, - namespace_id: rule.namespace_id, - provenance: rule.provenance || '', - })) + ? data.map((rule: Record) => mapAlertRule(rule)) : [], }, } @@ -86,15 +70,7 @@ export const listAlertRulesTool: ToolConfig< description: 'List of alert rules', items: { type: 'object', - properties: { - uid: { type: 'string', description: 'Alert rule UID' }, - title: { type: 'string', description: 'Alert rule title' }, - condition: { type: 'string', description: 'Alert condition' }, - folderUID: { type: 'string', description: 'Parent folder UID' }, - ruleGroup: { type: 'string', description: 'Rule group name' }, - noDataState: { type: 'string', description: 'State when no data is returned' }, - execErrState: { type: 'string', description: 'State on execution error' }, - }, + properties: ALERT_RULE_OUTPUT_FIELDS, }, }, }, diff --git a/apps/sim/tools/grafana/list_annotations.ts b/apps/sim/tools/grafana/list_annotations.ts index b267c3f236d..3c3c9af3cf0 100644 --- a/apps/sim/tools/grafana/list_annotations.ts +++ b/apps/sim/tools/grafana/list_annotations.ts @@ -46,9 +46,16 @@ export const listAnnotationsTool: ToolConfig< }, dashboardUid: { type: 'string', - required: true, + required: false, visibility: 'user-or-llm', - description: 'Dashboard UID to query annotations from (e.g., abc123def)', + description: + 'Dashboard UID to query annotations from (e.g., abc123def). Omit to query annotations across the organization.', + }, + dashboardId: { + type: 'number', + required: false, + visibility: 'user-only', + description: 'Legacy numeric dashboard ID filter (prefer dashboardUid)', }, panelId: { type: 'number', @@ -56,6 +63,18 @@ export const listAnnotationsTool: ToolConfig< visibility: 'user-or-llm', description: 'Filter by panel ID (e.g., 1, 2)', }, + alertId: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Filter by alert ID', + }, + userId: { + type: 'number', + required: false, + visibility: 'user-or-llm', + description: 'Filter by ID of the user who created the annotation', + }, tags: { type: 'string', required: false, @@ -84,7 +103,10 @@ export const listAnnotationsTool: ToolConfig< if (params.from) searchParams.set('from', String(params.from)) if (params.to) searchParams.set('to', String(params.to)) if (params.dashboardUid) searchParams.set('dashboardUID', params.dashboardUid) + if (params.dashboardId) searchParams.set('dashboardId', String(params.dashboardId)) if (params.panelId) searchParams.set('panelId', String(params.panelId)) + if (params.alertId) searchParams.set('alertId', String(params.alertId)) + if (params.userId) searchParams.set('userId', String(params.userId)) if (params.tags) { params.tags.split(',').forEach((t) => searchParams.append('tags', t.trim())) } @@ -109,27 +131,27 @@ export const listAnnotationsTool: ToolConfig< transformResponse: async (response: Response) => { const data = await response.json() - - // Handle potential nested array structure const rawAnnotations = Array.isArray(data) ? data.flat() : [] return { success: true, output: { - annotations: rawAnnotations.map((a: any) => ({ - id: a.id, - dashboardId: a.dashboardId, - dashboardUID: a.dashboardUID, - created: a.created, - updated: a.updated, - time: a.time, - timeEnd: a.timeEnd, - text: a.text, - tags: a.tags || [], - login: a.login, - email: a.email, - avatarUrl: a.avatarUrl, - data: a.data || {}, + annotations: rawAnnotations.map((a: Record) => ({ + id: (a.id as number) ?? null, + alertId: (a.alertId as number) ?? null, + dashboardId: (a.dashboardId as number) ?? null, + dashboardUID: (a.dashboardUID as string) ?? null, + panelId: (a.panelId as number) ?? null, + userId: (a.userId as number) ?? null, + userName: (a.userName as string) ?? null, + newState: (a.newState as string) ?? null, + prevState: (a.prevState as string) ?? null, + time: (a.time as number) ?? null, + timeEnd: (a.timeEnd as number) ?? null, + text: (a.text as string) ?? null, + metric: (a.metric as string) ?? null, + tags: (a.tags as string[]) ?? [], + data: (a.data as Record) ?? {}, })), }, } @@ -143,21 +165,36 @@ export const listAnnotationsTool: ToolConfig< type: 'object', properties: { id: { type: 'number', description: 'Annotation ID' }, - dashboardId: { type: 'number', description: 'Dashboard ID' }, - dashboardUID: { type: 'string', description: 'Dashboard UID' }, - created: { type: 'number', description: 'Creation timestamp in epoch ms' }, - updated: { type: 'number', description: 'Last update timestamp in epoch ms' }, + alertId: { type: 'number', description: 'Associated alert ID (0 if not alert-driven)' }, + dashboardId: { type: 'number', description: 'Dashboard ID', optional: true }, + dashboardUID: { type: 'string', description: 'Dashboard UID', optional: true }, + panelId: { type: 'number', description: 'Panel ID within the dashboard', optional: true }, + userId: { type: 'number', description: 'ID of the user who created the annotation' }, + userName: { + type: 'string', + description: 'Username of the user who created the annotation', + optional: true, + }, + newState: { + type: 'string', + description: 'New alert state (alert annotations only)', + optional: true, + }, + prevState: { + type: 'string', + description: 'Previous alert state (alert annotations only)', + optional: true, + }, time: { type: 'number', description: 'Start time in epoch ms' }, - timeEnd: { type: 'number', description: 'End time in epoch ms' }, + timeEnd: { type: 'number', description: 'End time in epoch ms', optional: true }, text: { type: 'string', description: 'Annotation text' }, - tags: { type: 'array', items: { type: 'string' }, description: 'Annotation tags' }, - login: { type: 'string', description: 'Login of the user who created the annotation' }, - email: { type: 'string', description: 'Email of the user who created the annotation' }, - avatarUrl: { type: 'string', description: 'Avatar URL of the user' }, - data: { - type: 'json', - description: 'Additional annotation data object from Grafana', + metric: { + type: 'string', + description: 'Metric associated with the annotation', + optional: true, }, + tags: { type: 'array', items: { type: 'string' }, description: 'Annotation tags' }, + data: { type: 'json', description: 'Additional annotation data object from Grafana' }, }, }, }, diff --git a/apps/sim/tools/grafana/list_contact_points.ts b/apps/sim/tools/grafana/list_contact_points.ts index 40e339e8ccc..cf044f21c68 100644 --- a/apps/sim/tools/grafana/list_contact_points.ts +++ b/apps/sim/tools/grafana/list_contact_points.ts @@ -32,10 +32,22 @@ export const listContactPointsTool: ToolConfig< visibility: 'user-or-llm', description: 'Organization ID for multi-org Grafana instances (e.g., 1, 2)', }, + name: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Filter contact points by exact name match', + }, }, request: { - url: (params) => `${params.baseUrl.replace(/\/$/, '')}/api/v1/provisioning/contact-points`, + url: (params) => { + const baseUrl = params.baseUrl.replace(/\/$/, '') + const searchParams = new URLSearchParams() + if (params.name) searchParams.set('name', params.name) + const queryString = searchParams.toString() + return `${baseUrl}/api/v1/provisioning/contact-points${queryString ? `?${queryString}` : ''}` + }, method: 'GET', headers: (params) => { const headers: Record = { @@ -56,13 +68,13 @@ export const listContactPointsTool: ToolConfig< success: true, output: { contactPoints: Array.isArray(data) - ? data.map((cp: any) => ({ - uid: cp.uid, - name: cp.name, - type: cp.type, - settings: cp.settings || {}, - disableResolveMessage: cp.disableResolveMessage || false, - provenance: cp.provenance || '', + ? data.map((cp: Record) => ({ + uid: (cp.uid as string) ?? null, + name: (cp.name as string) ?? null, + type: (cp.type as string) ?? null, + settings: (cp.settings as Record) ?? {}, + disableResolveMessage: (cp.disableResolveMessage as boolean) ?? false, + provenance: (cp.provenance as string) ?? '', })) : [], }, @@ -80,6 +92,14 @@ export const listContactPointsTool: ToolConfig< name: { type: 'string', description: 'Contact point name' }, type: { type: 'string', description: 'Notification type (email, slack, etc.)' }, settings: { type: 'object', description: 'Type-specific settings' }, + disableResolveMessage: { + type: 'boolean', + description: 'Whether resolve messages are disabled', + }, + provenance: { + type: 'string', + description: 'Provisioning source (empty if API-managed)', + }, }, }, }, diff --git a/apps/sim/tools/grafana/list_dashboards.ts b/apps/sim/tools/grafana/list_dashboards.ts index 855f008415a..39299199a54 100644 --- a/apps/sim/tools/grafana/list_dashboards.ts +++ b/apps/sim/tools/grafana/list_dashboards.ts @@ -44,11 +44,17 @@ export const listDashboardsTool: ToolConfig< visibility: 'user-or-llm', description: 'Filter by tag (comma-separated for multiple tags)', }, - folderIds: { + folderUIDs: { type: 'string', required: false, visibility: 'user-or-llm', - description: 'Filter by folder IDs (comma-separated, e.g., 1,2,3)', + description: 'Filter by folder UIDs (comma-separated, e.g., abc123,def456)', + }, + dashboardUIDs: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Filter by dashboard UIDs (comma-separated, e.g., abc123,def456)', }, starred: { type: 'boolean', @@ -60,7 +66,13 @@ export const listDashboardsTool: ToolConfig< type: 'number', required: false, visibility: 'user-only', - description: 'Maximum number of dashboards to return', + description: 'Maximum number of dashboards to return (default 1000)', + }, + page: { + type: 'number', + required: false, + visibility: 'user-only', + description: 'Page number for pagination (1-based)', }, }, @@ -74,11 +86,17 @@ export const listDashboardsTool: ToolConfig< if (params.tag) { params.tag.split(',').forEach((t) => searchParams.append('tag', t.trim())) } - if (params.folderIds) { - params.folderIds.split(',').forEach((id) => searchParams.append('folderIds', id.trim())) + if (params.folderUIDs) { + params.folderUIDs.split(',').forEach((uid) => searchParams.append('folderUIDs', uid.trim())) + } + if (params.dashboardUIDs) { + params.dashboardUIDs + .split(',') + .forEach((uid) => searchParams.append('dashboardUIDs', uid.trim())) } if (params.starred) searchParams.set('starred', 'true') if (params.limit) searchParams.set('limit', String(params.limit)) + if (params.page) searchParams.set('page', String(params.page)) return `${baseUrl}/api/search?${searchParams.toString()}` }, @@ -102,21 +120,19 @@ export const listDashboardsTool: ToolConfig< success: true, output: { dashboards: Array.isArray(data) - ? data.map((d: any) => ({ - id: d.id, - uid: d.uid, - title: d.title, - uri: d.uri, - url: d.url, - slug: d.slug, - type: d.type, - tags: d.tags || [], - isStarred: d.isStarred || false, - folderId: d.folderId, - folderUid: d.folderUid, - folderTitle: d.folderTitle, - folderUrl: d.folderUrl, - sortMeta: d.sortMeta, + ? data.map((d: Record) => ({ + id: (d.id as number) ?? null, + uid: (d.uid as string) ?? null, + title: (d.title as string) ?? null, + uri: (d.uri as string) ?? null, + url: (d.url as string) ?? null, + type: (d.type as string) ?? null, + tags: (d.tags as string[]) ?? [], + isStarred: (d.isStarred as boolean) ?? false, + folderId: (d.folderId as number) ?? null, + folderUid: (d.folderUid as string) ?? null, + folderTitle: (d.folderTitle as string) ?? null, + folderUrl: (d.folderUrl as string) ?? null, })) : [], }, diff --git a/apps/sim/tools/grafana/list_data_sources.ts b/apps/sim/tools/grafana/list_data_sources.ts index 55bd0ce4208..2c826248fce 100644 --- a/apps/sim/tools/grafana/list_data_sources.ts +++ b/apps/sim/tools/grafana/list_data_sources.ts @@ -56,22 +56,25 @@ export const listDataSourcesTool: ToolConfig< success: true, output: { dataSources: Array.isArray(data) - ? data.map((ds: any) => ({ - id: ds.id, - uid: ds.uid, - orgId: ds.orgId, - name: ds.name, - type: ds.type, - typeName: ds.typeName, - typeLogoUrl: ds.typeLogoUrl, - access: ds.access, - url: ds.url, - user: ds.user, - database: ds.database, - basicAuth: ds.basicAuth || false, - isDefault: ds.isDefault || false, - jsonData: ds.jsonData || {}, - readOnly: ds.readOnly || false, + ? data.map((ds: Record) => ({ + id: (ds.id as number) ?? null, + uid: (ds.uid as string) ?? null, + orgId: (ds.orgId as number) ?? null, + name: (ds.name as string) ?? null, + type: (ds.type as string) ?? null, + typeLogoUrl: (ds.typeLogoUrl as string) ?? null, + access: (ds.access as string) ?? null, + url: (ds.url as string) ?? null, + user: (ds.user as string) ?? null, + database: (ds.database as string) ?? null, + basicAuth: (ds.basicAuth as boolean) ?? false, + basicAuthUser: (ds.basicAuthUser as string) ?? null, + withCredentials: (ds.withCredentials as boolean) ?? null, + isDefault: (ds.isDefault as boolean) ?? false, + jsonData: (ds.jsonData as Record) ?? {}, + secureJsonFields: (ds.secureJsonFields as Record) ?? {}, + version: (ds.version as number) ?? null, + readOnly: (ds.readOnly as boolean) ?? false, })) : [], }, @@ -87,10 +90,34 @@ export const listDataSourcesTool: ToolConfig< properties: { id: { type: 'number', description: 'Data source ID' }, uid: { type: 'string', description: 'Data source UID' }, + orgId: { type: 'number', description: 'Organization ID' }, name: { type: 'string', description: 'Data source name' }, type: { type: 'string', description: 'Data source type (prometheus, mysql, etc.)' }, + typeLogoUrl: { type: 'string', description: 'Logo URL for the data source type' }, + access: { type: 'string', description: 'Access mode (proxy or direct)' }, url: { type: 'string', description: 'Data source URL' }, + user: { type: 'string', description: 'Username used to connect' }, + database: { type: 'string', description: 'Database name (if applicable)' }, + basicAuth: { type: 'boolean', description: 'Whether basic auth is enabled' }, + basicAuthUser: { + type: 'string', + description: 'Basic auth username', + optional: true, + }, + withCredentials: { + type: 'boolean', + description: 'Whether to send credentials with cross-origin requests', + optional: true, + }, isDefault: { type: 'boolean', description: 'Whether this is the default data source' }, + jsonData: { type: 'object', description: 'Type-specific JSON configuration' }, + secureJsonFields: { + type: 'object', + description: 'Map of secure fields that are set (values are not returned)', + optional: true, + }, + version: { type: 'number', description: 'Data source version', optional: true }, + readOnly: { type: 'boolean', description: 'Whether the data source is read-only' }, }, }, }, diff --git a/apps/sim/tools/grafana/list_folders.ts b/apps/sim/tools/grafana/list_folders.ts index 03aa569d672..85b1afc5500 100644 --- a/apps/sim/tools/grafana/list_folders.ts +++ b/apps/sim/tools/grafana/list_folders.ts @@ -38,6 +38,12 @@ export const listFoldersTool: ToolConfig ({ - id: f.id, - uid: f.uid, - title: f.title, - hasAcl: f.hasAcl || false, - canSave: f.canSave || false, - canEdit: f.canEdit || false, - canAdmin: f.canAdmin || false, - canDelete: f.canDelete || false, - createdBy: f.createdBy || '', - created: f.created || '', - updatedBy: f.updatedBy || '', - updated: f.updated || '', - version: f.version || 0, + ? data.map((f: Record) => ({ + id: (f.id as number) ?? null, + uid: (f.uid as string) ?? null, + title: (f.title as string) ?? null, + url: (f.url as string) ?? null, + parentUid: (f.parentUid as string) ?? null, + parents: (f.parents as { uid: string; title: string; url: string }[]) ?? [], + hasAcl: (f.hasAcl as boolean) ?? null, + canSave: (f.canSave as boolean) ?? null, + canEdit: (f.canEdit as boolean) ?? null, + canAdmin: (f.canAdmin as boolean) ?? null, + createdBy: (f.createdBy as string) ?? null, + created: (f.created as string) ?? null, + updatedBy: (f.updatedBy as string) ?? null, + updated: (f.updated as string) ?? null, + version: (f.version as number) ?? null, })) : [], }, @@ -101,19 +110,58 @@ export const listFoldersTool: ToolConfig = { + id: { type: 'number', description: 'Alert rule numeric ID', optional: true }, + uid: { type: 'string', description: 'Alert rule UID' }, + title: { type: 'string', description: 'Alert rule title' }, + condition: { type: 'string', description: 'RefId of the query used as the alert condition' }, + data: { type: 'json', description: 'Alert rule query/expression data array' }, + updated: { type: 'string', description: 'Last update timestamp', optional: true }, + noDataState: { type: 'string', description: 'State when no data is returned' }, + execErrState: { type: 'string', description: 'State on execution error' }, + for: { type: 'string', description: 'Duration the condition must hold before firing' }, + keepFiringFor: { + type: 'string', + description: 'Duration to keep firing after condition stops', + optional: true, + }, + missingSeriesEvalsToResolve: { + type: 'number', + description: 'Number of missing series evaluations before resolving', + optional: true, + }, + annotations: { type: 'json', description: 'Alert annotations' }, + labels: { type: 'json', description: 'Alert labels' }, + isPaused: { type: 'boolean', description: 'Whether the rule is paused' }, + folderUID: { type: 'string', description: 'Parent folder UID' }, + ruleGroup: { type: 'string', description: 'Rule group name' }, + orgID: { type: 'number', description: 'Organization ID' }, + provenance: { type: 'string', description: 'Provisioning source (empty if API-managed)' }, + notification_settings: { + type: 'json', + description: 'Per-rule notification settings (overrides)', + optional: true, + }, + record: { + type: 'json', + description: 'Recording rule configuration (recording rules only)', + optional: true, + }, +} // Common parameters for all Grafana tools interface GrafanaBaseParams { @@ -69,9 +111,9 @@ interface GrafanaDashboard { schemaVersion: number version: number refresh: string - panels: any[] - templating: any - annotations: any + panels: Record[] + templating: Record + annotations: Record time: { from: string to: string @@ -88,26 +130,26 @@ export interface GrafanaGetDashboardResponse extends ToolResponse { export interface GrafanaListDashboardsParams extends GrafanaBaseParams { query?: string tag?: string - folderIds?: string + folderUIDs?: string + dashboardUIDs?: string starred?: boolean limit?: number + page?: number } interface GrafanaDashboardSearchResult { - id: number - uid: string - title: string - uri: string - url: string - slug: string - type: string + id: number | null + uid: string | null + title: string | null + uri: string | null + url: string | null + type: string | null tags: string[] isStarred: boolean - folderId: number - folderUid: string - folderTitle: string - folderUrl: string - sortMeta: number + folderId: number | null + folderUid: string | null + folderTitle: string | null + folderUrl: string | null } export interface GrafanaListDashboardsResponse extends ToolResponse { @@ -177,23 +219,26 @@ export interface GrafanaDeleteDashboardResponse extends ToolResponse { export interface GrafanaListAlertRulesParams extends GrafanaBaseParams {} interface GrafanaAlertRule { - uid: string - title: string - condition: string - data: any[] - updated: string - noDataState: string - execErrState: string - for: string + id: number | null + uid: string | null + title: string | null + condition: string | null + data: unknown[] + updated: string | null + noDataState: string | null + execErrState: string | null + for: string | null + keepFiringFor: string | null + missingSeriesEvalsToResolve: number | null annotations: Record labels: Record isPaused: boolean - folderUID: string - ruleGroup: string - orgId: number - namespace_uid: string - namespace_id: number + folderUID: string | null + ruleGroup: string | null + orgID: number | null provenance: string + notification_settings: Record | null + record: Record | null } export interface GrafanaListAlertRulesResponse extends ToolResponse { @@ -214,13 +259,20 @@ export interface GrafanaCreateAlertRuleParams extends GrafanaBaseParams { title: string folderUid: string ruleGroup: string - condition: string + condition?: string data: string // JSON string of data array forDuration?: string noDataState?: string execErrState?: string annotations?: string // JSON string labels?: string // JSON string + uid?: string + isPaused?: boolean + keepFiringFor?: string + missingSeriesEvalsToResolve?: number + notificationSettings?: string // JSON string + record?: string // JSON string + disableProvenance?: boolean } export interface GrafanaCreateAlertRuleResponse extends ToolResponse { @@ -239,6 +291,12 @@ export interface GrafanaUpdateAlertRuleParams extends GrafanaBaseParams { execErrState?: string annotations?: string // JSON string labels?: string // JSON string + isPaused?: boolean + keepFiringFor?: string + missingSeriesEvalsToResolve?: number + notificationSettings?: string // JSON string + record?: string // JSON string + disableProvenance?: boolean } interface GrafanaUpdateAlertRuleResponse extends ToolResponse { @@ -266,19 +324,21 @@ export interface GrafanaCreateAnnotationParams extends GrafanaBaseParams { } interface GrafanaAnnotation { - id: number - dashboardId: number - dashboardUID: string - created: number - updated: number - time: number - timeEnd: number - text: string + id: number | null + alertId: number | null + dashboardId: number | null + dashboardUID: string | null + panelId: number | null + userId: number | null + userName: string | null + newState: string | null + prevState: string | null + time: number | null + timeEnd: number | null + text: string | null + metric: string | null tags: string[] - login: string - email: string - avatarUrl: string - data: any + data: Record } export interface GrafanaCreateAnnotationResponse extends ToolResponse { @@ -291,8 +351,11 @@ export interface GrafanaCreateAnnotationResponse extends ToolResponse { export interface GrafanaListAnnotationsParams extends GrafanaBaseParams { from?: number to?: number + dashboardId?: number dashboardUid?: string panelId?: number + alertId?: number + userId?: number tags?: string // comma-separated type?: string limit?: number @@ -306,7 +369,7 @@ export interface GrafanaListAnnotationsResponse extends ToolResponse { export interface GrafanaUpdateAnnotationParams extends GrafanaBaseParams { annotationId: number - text: string + text?: string tags?: string // comma-separated time?: number timeEnd?: number @@ -338,15 +401,18 @@ interface GrafanaDataSource { orgId: number name: string type: string - typeName: string typeLogoUrl: string access: string url: string user: string database: string basicAuth: boolean + basicAuthUser?: string + withCredentials?: boolean isDefault: boolean - jsonData: any + jsonData: Record + secureJsonFields?: Record + version?: number readOnly: boolean } @@ -368,22 +434,31 @@ export interface GrafanaGetDataSourceResponse extends ToolResponse { export interface GrafanaListFoldersParams extends GrafanaBaseParams { limit?: number page?: number + parentUid?: string +} + +interface GrafanaFolderParent { + uid: string + title: string + url: string } interface GrafanaFolder { id: number uid: string title: string - hasAcl: boolean - canSave: boolean - canEdit: boolean - canAdmin: boolean - canDelete: boolean - createdBy: string - created: string - updatedBy: string - updated: string - version: number + url?: string + hasAcl?: boolean + canSave?: boolean + canEdit?: boolean + canAdmin?: boolean + createdBy?: string + created?: string + updatedBy?: string + updated?: string + version?: number + parentUid?: string | null + parents?: GrafanaFolderParent[] } export interface GrafanaListFoldersResponse extends ToolResponse { @@ -395,6 +470,7 @@ export interface GrafanaListFoldersResponse extends ToolResponse { export interface GrafanaCreateFolderParams extends GrafanaBaseParams { title: string uid?: string + parentUid?: string } export interface GrafanaCreateFolderResponse extends ToolResponse { @@ -402,13 +478,15 @@ export interface GrafanaCreateFolderResponse extends ToolResponse { } // Contact Points types -export interface GrafanaListContactPointsParams extends GrafanaBaseParams {} +export interface GrafanaListContactPointsParams extends GrafanaBaseParams { + name?: string +} interface GrafanaContactPoint { uid: string name: string type: string - settings: Record + settings: Record disableResolveMessage: boolean provenance: string } diff --git a/apps/sim/tools/grafana/update_alert_rule.ts b/apps/sim/tools/grafana/update_alert_rule.ts index 386a9b0e825..9ca23bff773 100644 --- a/apps/sim/tools/grafana/update_alert_rule.ts +++ b/apps/sim/tools/grafana/update_alert_rule.ts @@ -1,4 +1,5 @@ -import type { GrafanaUpdateAlertRuleParams } from '@/tools/grafana/types' +import { ALERT_RULE_OUTPUT_FIELDS, type GrafanaUpdateAlertRuleParams } from '@/tools/grafana/types' +import { mapAlertRule } from '@/tools/grafana/utils' import type { ToolConfig, ToolResponse } from '@/tools/types' // Using ToolResponse for intermediate state since this tool fetches existing data first @@ -79,7 +80,7 @@ export const updateAlertRuleTool: ToolConfig = { + const updatedRule: Record = { ...existingRule, } @@ -148,12 +185,45 @@ export const updateAlertRuleTool: ToolConfig { - const body: Record = { - text: params.text, - } + const body: Record = {} + + if (params.text !== undefined) body.text = params.text + if (params.time) body.time = params.time + if (params.timeEnd) body.timeEnd = params.timeEnd if (params.tags) { body.tags = params.tags @@ -89,14 +91,6 @@ export const updateAnnotationTool: ToolConfig< .filter((t) => t) } - if (params.time) { - body.time = params.time - } - - if (params.timeEnd) { - body.timeEnd = params.timeEnd - } - return body }, }, diff --git a/apps/sim/tools/grafana/update_dashboard.ts b/apps/sim/tools/grafana/update_dashboard.ts index 2913878012f..23449f36830 100644 --- a/apps/sim/tools/grafana/update_dashboard.ts +++ b/apps/sim/tools/grafana/update_dashboard.ts @@ -74,7 +74,8 @@ export const updateDashboardTool: ToolConfig = { dashboard: updatedDashboard, - overwrite: params.overwrite !== false, + overwrite: params.overwrite === true, } // Use existing folder if not specified diff --git a/apps/sim/tools/grafana/utils.ts b/apps/sim/tools/grafana/utils.ts new file mode 100644 index 00000000000..1d70b0703d3 --- /dev/null +++ b/apps/sim/tools/grafana/utils.ts @@ -0,0 +1,31 @@ +/** + * Map a raw Grafana ProvisionedAlertRule JSON object to the canonical output shape + * shared across list/get/create/update alert rule tools. + */ +export function mapAlertRule(rule: Record) { + return { + id: (rule.id as number) ?? null, + uid: (rule.uid as string) ?? null, + title: (rule.title as string) ?? null, + condition: (rule.condition as string) ?? null, + data: (rule.data as unknown[]) ?? [], + updated: (rule.updated as string) ?? null, + noDataState: (rule.noDataState as string) ?? null, + execErrState: (rule.execErrState as string) ?? null, + for: (rule.for as string) ?? null, + keepFiringFor: (rule.keep_firing_for as string) ?? (rule.keepFiringFor as string) ?? null, + missingSeriesEvalsToResolve: + (rule.missingSeriesEvalsToResolve as number) ?? + (rule.missing_series_evals_to_resolve as number) ?? + null, + annotations: (rule.annotations as Record) ?? {}, + labels: (rule.labels as Record) ?? {}, + isPaused: (rule.isPaused as boolean) ?? false, + folderUID: (rule.folderUID as string) ?? null, + ruleGroup: (rule.ruleGroup as string) ?? null, + orgID: (rule.orgID as number) ?? (rule.orgId as number) ?? null, + provenance: (rule.provenance as string) ?? '', + notification_settings: (rule.notification_settings as Record) ?? null, + record: (rule.record as Record) ?? null, + } +} diff --git a/apps/sim/tools/supabase/count.ts b/apps/sim/tools/supabase/count.ts index 88a7bc0cf13..7e5d2c0f3ad 100644 --- a/apps/sim/tools/supabase/count.ts +++ b/apps/sim/tools/supabase/count.ts @@ -1,3 +1,4 @@ +import { validateDatabaseIdentifier } from '@/lib/core/security/input-validation' import type { SupabaseCountParams, SupabaseCountResponse } from '@/tools/supabase/types' import { supabaseBaseUrl } from '@/tools/supabase/utils' import type { ToolConfig } from '@/tools/types' @@ -50,9 +51,10 @@ export const countTool: ToolConfig = request: { url: (params) => { - let url = `${supabaseBaseUrl(params.projectId)}/rest/v1/${params.table}?select=*` + const tableValidation = validateDatabaseIdentifier(params.table, 'table') + if (!tableValidation.isValid) throw new Error(tableValidation.error) + let url = `${supabaseBaseUrl(params.projectId)}/rest/v1/${encodeURIComponent(params.table)}?select=*` - // Add filters if provided if (params.filter?.trim()) { url += `&${params.filter.trim()}` } diff --git a/apps/sim/tools/supabase/delete.ts b/apps/sim/tools/supabase/delete.ts index a76a1781b1c..967229868e1 100644 --- a/apps/sim/tools/supabase/delete.ts +++ b/apps/sim/tools/supabase/delete.ts @@ -1,3 +1,4 @@ +import { validateDatabaseIdentifier } from '@/lib/core/security/input-validation' import type { SupabaseDeleteParams, SupabaseDeleteResponse } from '@/tools/supabase/types' import { supabaseBaseUrl } from '@/tools/supabase/utils' import type { ToolConfig } from '@/tools/types' @@ -44,10 +45,11 @@ export const deleteTool: ToolConfig { - // Construct the URL for the Supabase REST API with select to return deleted data - let url = `${supabaseBaseUrl(params.projectId)}/rest/v1/${params.table}?select=*` + const tableValidation = validateDatabaseIdentifier(params.table, 'table') + if (!tableValidation.isValid) throw new Error(tableValidation.error) + + let url = `${supabaseBaseUrl(params.projectId)}/rest/v1/${encodeURIComponent(params.table)}?select=*` - // Add filters (required for delete) - using PostgREST syntax if (params.filter?.trim()) { url += `&${params.filter.trim()}` } else { diff --git a/apps/sim/tools/supabase/get_row.ts b/apps/sim/tools/supabase/get_row.ts index e21414dee2c..dec54e3d583 100644 --- a/apps/sim/tools/supabase/get_row.ts +++ b/apps/sim/tools/supabase/get_row.ts @@ -1,3 +1,4 @@ +import { validateDatabaseIdentifier } from '@/lib/core/security/input-validation' import type { SupabaseGetRowParams, SupabaseGetRowResponse } from '@/tools/supabase/types' import { supabaseBaseUrl } from '@/tools/supabase/utils' import type { ToolConfig } from '@/tools/types' @@ -50,16 +51,16 @@ export const getRowTool: ToolConfig { - // Construct the URL for the Supabase REST API + const tableValidation = validateDatabaseIdentifier(params.table, 'table') + if (!tableValidation.isValid) throw new Error(tableValidation.error) + const selectColumns = params.select?.trim() || '*' - let url = `${supabaseBaseUrl(params.projectId)}/rest/v1/${params.table}?select=${encodeURIComponent(selectColumns)}` + let url = `${supabaseBaseUrl(params.projectId)}/rest/v1/${encodeURIComponent(params.table)}?select=${encodeURIComponent(selectColumns)}` - // Add filters (required for get_row) - using PostgREST syntax if (params.filter?.trim()) { url += `&${params.filter.trim()}` } - // Limit to 1 row since we want a single row url += `&limit=1` return url diff --git a/apps/sim/tools/supabase/insert.ts b/apps/sim/tools/supabase/insert.ts index 9cd33696536..39dc4264e54 100644 --- a/apps/sim/tools/supabase/insert.ts +++ b/apps/sim/tools/supabase/insert.ts @@ -1,3 +1,4 @@ +import { validateDatabaseIdentifier } from '@/lib/core/security/input-validation' import type { SupabaseInsertParams, SupabaseInsertResponse } from '@/tools/supabase/types' import { supabaseBaseUrl } from '@/tools/supabase/utils' import type { ToolConfig } from '@/tools/types' @@ -43,7 +44,11 @@ export const insertTool: ToolConfig `${supabaseBaseUrl(params.projectId)}/rest/v1/${params.table}?select=*`, + url: (params) => { + const tableValidation = validateDatabaseIdentifier(params.table, 'table') + if (!tableValidation.isValid) throw new Error(tableValidation.error) + return `${supabaseBaseUrl(params.projectId)}/rest/v1/${encodeURIComponent(params.table)}?select=*` + }, method: 'POST', headers: (params) => { const headers: Record = { diff --git a/apps/sim/tools/supabase/query.ts b/apps/sim/tools/supabase/query.ts index 46847c07146..0fcf1b75dda 100644 --- a/apps/sim/tools/supabase/query.ts +++ b/apps/sim/tools/supabase/query.ts @@ -1,3 +1,4 @@ +import { validateDatabaseIdentifier } from '@/lib/core/security/input-validation' import type { SupabaseQueryParams, SupabaseQueryResponse } from '@/tools/supabase/types' import { supabaseBaseUrl } from '@/tools/supabase/utils' import type { ToolConfig } from '@/tools/types' @@ -68,9 +69,10 @@ export const queryTool: ToolConfig = request: { url: (params) => { - // Construct the URL for the Supabase REST API + const tableValidation = validateDatabaseIdentifier(params.table, 'table') + if (!tableValidation.isValid) throw new Error(tableValidation.error) const selectColumns = params.select?.trim() || '*' - let url = `${supabaseBaseUrl(params.projectId)}/rest/v1/${params.table}?select=${encodeURIComponent(selectColumns)}` + let url = `${supabaseBaseUrl(params.projectId)}/rest/v1/${encodeURIComponent(params.table)}?select=${encodeURIComponent(selectColumns)}` // Add filters if provided - using PostgREST syntax if (params.filter?.trim()) { diff --git a/apps/sim/tools/supabase/text_search.ts b/apps/sim/tools/supabase/text_search.ts index 5a813c0c7a7..aa7e7402b6a 100644 --- a/apps/sim/tools/supabase/text_search.ts +++ b/apps/sim/tools/supabase/text_search.ts @@ -1,3 +1,4 @@ +import { validateDatabaseIdentifier } from '@/lib/core/security/input-validation' import type { SupabaseTextSearchParams, SupabaseTextSearchResponse } from '@/tools/supabase/types' import { supabaseBaseUrl } from '@/tools/supabase/utils' import type { ToolConfig } from '@/tools/types' @@ -74,11 +75,12 @@ export const textSearchTool: ToolConfig { + const tableValidation = validateDatabaseIdentifier(params.table, 'table') + if (!tableValidation.isValid) throw new Error(tableValidation.error) const searchType = params.searchType || 'websearch' const language = params.language || 'english' - // Build the text search filter - let url = `${supabaseBaseUrl(params.projectId)}/rest/v1/${params.table}?select=*` + let url = `${supabaseBaseUrl(params.projectId)}/rest/v1/${encodeURIComponent(params.table)}?select=*` // Map search types to PostgREST operators // plfts = plainto_tsquery (natural language), phfts = phraseto_tsquery, wfts = websearch_to_tsquery diff --git a/apps/sim/tools/supabase/update.ts b/apps/sim/tools/supabase/update.ts index f6aad13ed71..28c26e53d89 100644 --- a/apps/sim/tools/supabase/update.ts +++ b/apps/sim/tools/supabase/update.ts @@ -1,3 +1,4 @@ +import { validateDatabaseIdentifier } from '@/lib/core/security/input-validation' import type { SupabaseUpdateParams, SupabaseUpdateResponse } from '@/tools/supabase/types' import { supabaseBaseUrl } from '@/tools/supabase/utils' import type { ToolConfig } from '@/tools/types' @@ -50,12 +51,17 @@ export const updateTool: ToolConfig { - // Construct the URL for the Supabase REST API with select to return updated data - let url = `${supabaseBaseUrl(params.projectId)}/rest/v1/${params.table}?select=*` + const tableValidation = validateDatabaseIdentifier(params.table, 'table') + if (!tableValidation.isValid) throw new Error(tableValidation.error) + + let url = `${supabaseBaseUrl(params.projectId)}/rest/v1/${encodeURIComponent(params.table)}?select=*` - // Add filters (required for update) - using PostgREST syntax if (params.filter?.trim()) { url += `&${params.filter.trim()}` + } else { + throw new Error( + 'Filter is required for update operations to prevent accidental update of all rows' + ) } return url diff --git a/apps/sim/tools/supabase/upsert.ts b/apps/sim/tools/supabase/upsert.ts index d7ebf41a7e5..8b0fe7213dc 100644 --- a/apps/sim/tools/supabase/upsert.ts +++ b/apps/sim/tools/supabase/upsert.ts @@ -1,3 +1,4 @@ +import { validateDatabaseIdentifier } from '@/lib/core/security/input-validation' import type { SupabaseUpsertParams, SupabaseUpsertResponse } from '@/tools/supabase/types' import { supabaseBaseUrl } from '@/tools/supabase/utils' import type { ToolConfig } from '@/tools/types' @@ -43,7 +44,11 @@ export const upsertTool: ToolConfig `${supabaseBaseUrl(params.projectId)}/rest/v1/${params.table}?select=*`, + url: (params) => { + const tableValidation = validateDatabaseIdentifier(params.table, 'table') + if (!tableValidation.isValid) throw new Error(tableValidation.error) + return `${supabaseBaseUrl(params.projectId)}/rest/v1/${encodeURIComponent(params.table)}?select=*` + }, method: 'POST', headers: (params) => { const headers: Record = { diff --git a/bun.lock b/bun.lock index a03ee598a61..06d495404cd 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "simstudio", diff --git a/helm/sim/.claude/skills/sim-helm/references/values-model.md b/helm/sim/.claude/skills/sim-helm/references/values-model.md index 8e75492909a..801dfd77b7f 100644 --- a/helm/sim/.claude/skills/sim-helm/references/values-model.md +++ b/helm/sim/.claude/skills/sim-helm/references/values-model.md @@ -44,11 +44,9 @@ The Sim chart splits configuration across **four** layers. Understanding which l ## Why this layering exists -**Single source of truth per concern.** Secrets live in a Secret. Operational defaults live where users can override them. Chart-computed values live where the chart can authoritatively compute them. +**ESO compatibility.** When `externalSecrets.enabled=true`, the chart-managed Secret is **not rendered** — ESO renders one instead. Anything in Layer 1 must be mapped via `remoteRefs.app.` or it's silently missing. Layers 2–4 are unaffected by ESO. -**ESO compatibility.** When `externalSecrets.enabled=true`, the chart-managed Secret is **not rendered** — ESO renders one instead. Anything in Layer 1 must be mapped via `remoteRefs.app.` or it's silently missing. Layers 2–4 are unaffected by ESO. Putting operational tunables in `envDefaults` instead of `env` means ESO users don't have to map dozens of tunables — just the real secrets. - -**Backwards compatibility.** Layer 2 was added in chart 1.0.0 (formerly all defaults lived in `app.env`). The override-skip logic in the Deployment template means existing users who set values in `app.env` continue to work — those values win over `envDefaults`. +**Override precedence.** Values set in `app.env` (Layer 1 overrides) win over `envDefaults` (Layer 2) — so users who already had operational tunables in `app.env` continue to work. ## Where keys live — the canonical list diff --git a/helm/sim/README.md b/helm/sim/README.md index 80fc5b81f1c..6bb45abc8d0 100644 --- a/helm/sim/README.md +++ b/helm/sim/README.md @@ -217,7 +217,7 @@ Before installing in production, confirm each of the following: * **Pinned images** — override `image.tag` (or `image.digest`) with an explicit version. Do not rely on the chart's default tag in production. * **Secrets management** — provide secrets via External Secrets Operator (ESO) or pre-created Kubernetes Secrets. Never commit secrets to `values.yaml`. * **TLS / Ingress** — set the `cert-manager.io/cluster-issuer` annotation on the ingress and tune `proxy-body-size` / `proxy-read-timeout` for your workload. See commented examples in `values.yaml`. -* **Network policy egress** — review `networkPolicy.egress.exceptCidrs`. Defaults block cloud metadata endpoints (`169.254.169.254/32`, `169.254.170.2/32`); add your cluster's API server CIDR for stronger isolation. +* **Network policy egress** — review `networkPolicy.egressExceptCidrs`. Defaults block cloud metadata endpoints (`169.254.169.254/32`, `169.254.170.2/32`); add your cluster's API server CIDR for stronger isolation. Custom egress rules go in `networkPolicy.egress` (a list). * **Namespace hardening** — label the install namespace with Pod Security Standards `restricted` enforcement (`pod-security.kubernetes.io/enforce=restricted`). * **Env validation** — keys under `app.env`, `realtime.env`, and `copilot.env` are passed through to the application and validated at startup. The JSON Schema intentionally does not enforce `additionalProperties: false` (would break custom user envs), so typos like `OPENA_API_KEY` (instead of `OPENAI_API_KEY`) surface as missing-key errors at runtime, not at `helm install` time. Review your env block carefully. * **Set public URLs** — `app.env.NEXT_PUBLIC_APP_URL` and `app.env.BETTER_AUTH_URL` must match your public origin (e.g. `https://sim.example.com`). Leaving them as `localhost` breaks sign-in. diff --git a/helm/sim/templates/NOTES.txt b/helm/sim/templates/NOTES.txt index e563117500c..27be4b0b2a7 100644 --- a/helm/sim/templates/NOTES.txt +++ b/helm/sim/templates/NOTES.txt @@ -81,7 +81,16 @@ Your release is named {{ .Release.Name }} in namespace {{ .Release.Namespace }}. # Upgrade after changing values helm upgrade {{ .Release.Name }} ./helm/sim --namespace {{ .Release.Namespace }} -f your-values.yaml -5. Where to go next: +5. Upgrade notes (read before upgrading from a chart version released before this one): + + * externalSecrets.apiVersion default is "v1beta1" (was "v1"). v1beta1 is + supported by every ESO release from v0.7+ through current. If you're on + ESO v0.17+ and want the graduated v1 API, set externalSecrets.apiVersion: "v1". + * networkPolicy.egress remains a list of custom egress rules (unchanged). + Cloud-metadata CIDR blocking is now configured via networkPolicy.egressExceptCidrs + (defaults to AWS/GCP/Azure IMDS + ECS task metadata). + +6. Where to go next: * Production checklist: helm/sim/README.md (search "Production checklist") * Troubleshooting: helm/sim/README.md (search "Troubleshooting") diff --git a/helm/sim/templates/networkpolicy.yaml b/helm/sim/templates/networkpolicy.yaml index 4a19ae89377..a6db889d74a 100644 --- a/helm/sim/templates/networkpolicy.yaml +++ b/helm/sim/templates/networkpolicy.yaml @@ -107,14 +107,14 @@ spec: - ipBlock: cidr: 0.0.0.0/0 except: - {{- range (default (list "169.254.169.254/32" "169.254.170.2/32") .Values.networkPolicy.egress.exceptCidrs) }} + {{- range (default (list "169.254.169.254/32" "169.254.170.2/32") .Values.networkPolicy.egressExceptCidrs) }} - {{ . | quote }} {{- end }} ports: - protocol: TCP port: 443 # Allow custom egress rules - {{- with .Values.networkPolicy.egress.extraRules }} + {{- with .Values.networkPolicy.egress }} {{- toYaml . | nindent 2 }} {{- end }} @@ -189,14 +189,14 @@ spec: - ipBlock: cidr: 0.0.0.0/0 except: - {{- range (default (list "169.254.169.254/32" "169.254.170.2/32") .Values.networkPolicy.egress.exceptCidrs) }} + {{- range (default (list "169.254.169.254/32" "169.254.170.2/32") .Values.networkPolicy.egressExceptCidrs) }} - {{ . | quote }} {{- end }} ports: - protocol: TCP port: 443 # Allow custom egress rules - {{- with .Values.networkPolicy.egress.extraRules }} + {{- with .Values.networkPolicy.egress }} {{- toYaml . | nindent 2 }} {{- end }} {{- end }} @@ -296,11 +296,96 @@ spec: - ipBlock: cidr: 0.0.0.0/0 except: - {{- range (default (list "169.254.169.254/32" "169.254.170.2/32") .Values.networkPolicy.egress.exceptCidrs) }} + {{- range (default (list "169.254.169.254/32" "169.254.170.2/32") .Values.networkPolicy.egressExceptCidrs) }} - {{ . | quote }} {{- end }} ports: - protocol: TCP port: 443 {{- end }} + +{{- if .Values.telemetry.enabled }} +--- +# Network Policy for OpenTelemetry Collector +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "sim.fullname" . }}-otel-collector + namespace: {{ .Release.Namespace }} + labels: + {{- include "sim.labels" . | nindent 4 }} + app.kubernetes.io/component: telemetry +spec: + podSelector: + matchLabels: + {{- include "sim.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: telemetry + policyTypes: + - Ingress + - Egress + ingress: + # OTLP from app + - from: + - podSelector: + matchLabels: + {{- include "sim.app.selectorLabels" . | nindent 10 }} + ports: + - protocol: TCP + port: 4317 + - protocol: TCP + port: 4318 + # OTLP from realtime + {{- if .Values.realtime.enabled }} + - from: + - podSelector: + matchLabels: + {{- include "sim.realtime.selectorLabels" . | nindent 10 }} + ports: + - protocol: TCP + port: 4317 + - protocol: TCP + port: 4318 + {{- end }} + # OTLP from copilot + {{- if .Values.copilot.enabled }} + - from: + - podSelector: + matchLabels: + {{- include "sim.selectorLabels" . | nindent 10 }} + app.kubernetes.io/component: copilot + ports: + - protocol: TCP + port: 4317 + - protocol: TCP + port: 4318 + {{- end }} + egress: + # DNS + - to: [] + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + # HTTPS for forwarding to external observability backends (Datadog, Honeycomb, etc.) + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + {{- range (default (list "169.254.169.254/32" "169.254.170.2/32") .Values.networkPolicy.egressExceptCidrs) }} + - {{ . | quote }} + {{- end }} + ports: + - protocol: TCP + port: 443 +{{- end }} + +{{- /* + Copilot + copilot-postgresql intentionally do NOT ship dedicated NetworkPolicies. + Copilot requires REDIS_URL (external Redis on a non-443 port), and the chart + cannot know the user's Redis host/port at render time — a default egress rule + would silently block Redis on most installs. Users running networkPolicy.enabled=true + with copilot enabled should add their own NPs (or extend networkPolicy.egress + with the appropriate egress rules). +*/}} {{- end }} \ No newline at end of file diff --git a/helm/sim/templates/statefulset-copilot-postgres.yaml b/helm/sim/templates/statefulset-copilot-postgres.yaml index 68275fb3e9e..91dd5bad195 100644 --- a/helm/sim/templates/statefulset-copilot-postgres.yaml +++ b/helm/sim/templates/statefulset-copilot-postgres.yaml @@ -66,7 +66,11 @@ metadata: {{- include "sim.labels" . | nindent 4 }} app.kubernetes.io/component: copilot-postgresql spec: - serviceName: {{ include "sim.fullname" . }}-copilot-postgresql-headless + # Must remain {{ include "sim.fullname" . }}-copilot-postgresql (not the + # -headless name) — spec.serviceName is immutable on a StatefulSet, and + # the prior chart shipped with this value. Same rationale as the main + # postgresql STS; see statefulset-postgresql.yaml for details. + serviceName: {{ include "sim.fullname" . }}-copilot-postgresql replicas: 1 podManagementPolicy: OrderedReady updateStrategy: @@ -111,6 +115,10 @@ spec: envFrom: - secretRef: name: {{ include "sim.fullname" . }}-copilot-postgresql-secret + {{- if .Values.copilot.postgresql.startupProbe }} + startupProbe: + {{- toYaml .Values.copilot.postgresql.startupProbe | nindent 12 }} + {{- end }} {{- if .Values.copilot.postgresql.livenessProbe }} livenessProbe: {{- toYaml .Values.copilot.postgresql.livenessProbe | nindent 12 }} diff --git a/helm/sim/templates/statefulset-postgresql.yaml b/helm/sim/templates/statefulset-postgresql.yaml index e2a9bf402a2..50c78b8a03d 100644 --- a/helm/sim/templates/statefulset-postgresql.yaml +++ b/helm/sim/templates/statefulset-postgresql.yaml @@ -90,7 +90,12 @@ metadata: labels: {{- include "sim.postgresql.labels" . | nindent 4 }} spec: - serviceName: {{ include "sim.fullname" . }}-postgresql-headless + # Must remain {{ include "sim.fullname" . }}-postgresql (not the -headless + # name) — spec.serviceName is immutable on a StatefulSet, and the prior + # chart shipped with this value. Changing it would break `helm upgrade` for + # every existing install with `Forbidden: updates to statefulset spec ...`. + # The headless Service in services.yaml is added alongside, not as a swap. + serviceName: {{ include "sim.fullname" . }}-postgresql replicas: 1 minReadySeconds: 10 podManagementPolicy: OrderedReady @@ -135,6 +140,10 @@ spec: name: {{ include "sim.fullname" . }}-postgresql-env - secretRef: name: {{ include "sim.postgresqlSecretName" . }} + {{- if .Values.postgresql.startupProbe }} + startupProbe: + {{- toYaml .Values.postgresql.startupProbe | nindent 12 }} + {{- end }} {{- if .Values.postgresql.livenessProbe }} livenessProbe: {{- toYaml .Values.postgresql.livenessProbe | nindent 12 }} diff --git a/helm/sim/tests/networkpolicy_test.yaml b/helm/sim/tests/networkpolicy_test.yaml index da52d46f06a..d6e0b260ec4 100644 --- a/helm/sim/tests/networkpolicy_test.yaml +++ b/helm/sim/tests/networkpolicy_test.yaml @@ -128,10 +128,23 @@ tests: - protocol: TCP port: 3000 - - it: egress.extraRules are appended to both app and realtime NetworkPolicies + - it: telemetry collector NetworkPolicy renders when telemetry.enabled=true set: <<: *defaults - networkPolicy.egress.extraRules: + telemetry.enabled: true + documentIndex: 3 + asserts: + - equal: + path: kind + value: NetworkPolicy + - equal: + path: metadata.name + value: t-sim-otel-collector + + - it: networkPolicy.egress (custom rules) are appended to the app NetworkPolicy + set: + <<: *defaults + networkPolicy.egress: - to: [] ports: - protocol: TCP @@ -145,3 +158,21 @@ tests: ports: - protocol: TCP port: 5432 + + - it: networkPolicy.egress (custom rules) are appended to the realtime NetworkPolicy + set: + <<: *defaults + networkPolicy.egress: + - to: [] + ports: + - protocol: TCP + port: 5432 + documentIndex: 1 + asserts: + - contains: + path: spec.egress + content: + to: [] + ports: + - protocol: TCP + port: 5432 diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index a913c4b8366..e253f1a0af8 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -622,12 +622,22 @@ postgresql: targetPort: 5432 # Health checks + # startupProbe shields liveness from slow first-boot scenarios (pgvector + # extension init, WAL replay after a crash on a large data dir). Gives + # postgres up to 150s (30 * 5s) to become ready before liveness takes over. + startupProbe: + exec: + command: ["pg_isready", "-U", "postgres", "-d", "sim"] + periodSeconds: 5 + failureThreshold: 30 + timeoutSeconds: 5 + livenessProbe: exec: command: ["pg_isready", "-U", "postgres", "-d", "sim"] initialDelaySeconds: 10 periodSeconds: 5 - + readinessProbe: exec: command: ["pg_isready", "-U", "postgres", "-d", "sim"] @@ -954,7 +964,7 @@ monitoring: # to each other and to required external services (DNS, HTTPS) while blocking # everything else. The egress block additionally blacklists cloud metadata # endpoints (169.254.169.254/32, 169.254.170.2/32) by default — extend -# egress.exceptCidrs with your cluster's API server CIDR for tighter isolation. +# egressExceptCidrs with your cluster's API server CIDR for tighter isolation. # Your CNI must support NetworkPolicy (Calico, Cilium, GKE Dataplane V2, etc.). networkPolicy: enabled: false @@ -973,16 +983,18 @@ networkPolicy: # Custom ingress rules appended to the policy ingress: [] - # Egress configuration - egress: - # CIDRs excluded from broad HTTPS (443) egress. - # Defaults block AWS/GCP/Azure IMDS (169.254.169.254/32) and ECS task metadata - # (169.254.170.2/32). Add your cluster's API server CIDR for stronger isolation. - exceptCidrs: - - "169.254.169.254/32" - - "169.254.170.2/32" - # Custom egress rules appended to the policy - extraRules: [] + # Custom egress rules appended to the policy. + # Kept as a top-level list (not a map) for backward compatibility with the + # pre-1.0 chart that shipped `networkPolicy.egress: []`. Existing values + # files continue to work without changes. + egress: [] + + # CIDRs excluded from broad HTTPS (443) egress. + # Defaults block AWS/GCP/Azure IMDS (169.254.169.254/32) and ECS task metadata + # (169.254.170.2/32). Add your cluster's API server CIDR for stronger isolation. + egressExceptCidrs: + - "169.254.169.254/32" + - "169.254.170.2/32" # Shared storage for enterprise workflows requiring data sharing between pods sharedStorage: @@ -1438,6 +1450,16 @@ copilot: targetPort: 5432 # Health checks + # startupProbe shields liveness from slow first-boot scenarios (pgvector + # extension init, WAL replay after a crash). Gives postgres up to 150s + # (30 * 5s) to become ready before liveness takes over. + startupProbe: + exec: + command: ["pg_isready", "-U", "copilot", "-d", "copilot"] + periodSeconds: 5 + failureThreshold: 30 + timeoutSeconds: 5 + livenessProbe: exec: command: ["pg_isready", "-U", "copilot", "-d", "copilot"] @@ -1445,7 +1467,7 @@ copilot: periodSeconds: 5 timeoutSeconds: 5 failureThreshold: 10 - + readinessProbe: exec: command: ["pg_isready", "-U", "copilot", "-d", "copilot"] diff --git a/packages/python-sdk/README.md b/packages/python-sdk/README.md index e193e951c13..2690f635a17 100644 --- a/packages/python-sdk/README.md +++ b/packages/python-sdk/README.md @@ -115,17 +115,17 @@ result = client.execute_workflow_sync("workflow-id", {"data": "some input"}, tim **Returns:** `WorkflowExecutionResult` -##### get_job_status(task_id) +##### get_job_status(job_id) Get the status of an async job. ```python -status = client.get_job_status("task-id-from-async-execution") +status = client.get_job_status("job-id-from-async-execution") print("Job status:", status) ``` **Parameters:** -- `task_id` (str): The task ID returned from async execution +- `job_id` (str): The job ID returned from async execution **Returns:** `dict` @@ -248,10 +248,11 @@ class SimStudioError(Exception): @dataclass class AsyncExecutionResult: success: bool - task_id: str - status: str # 'queued' - created_at: str - links: Dict[str, str] + job_id: str + status_url: str + execution_id: Optional[str] = None + message: str = "" + async_execution: bool = True ``` ### RateLimitInfo diff --git a/packages/python-sdk/simstudio/__init__.py b/packages/python-sdk/simstudio/__init__.py index ec242338ec5..0e2609e2f26 100644 --- a/packages/python-sdk/simstudio/__init__.py +++ b/packages/python-sdk/simstudio/__init__.py @@ -49,10 +49,11 @@ class WorkflowStatus: class AsyncExecutionResult: """Result of an async workflow execution.""" success: bool - task_id: str - status: str # 'queued' - created_at: str - links: Dict[str, str] + job_id: str + status_url: str + execution_id: Optional[str] = None + message: str = "" + async_execution: bool = True @dataclass @@ -237,13 +238,14 @@ def execute_workflow( result_data = response.json() # Check if this is an async execution response (202 status) - if response.status_code == 202 and 'taskId' in result_data: + if response.status_code == 202 and 'jobId' in result_data: return AsyncExecutionResult( success=result_data.get('success', True), - task_id=result_data['taskId'], - status=result_data.get('status', 'queued'), - created_at=result_data.get('createdAt', ''), - links=result_data.get('links', {}) + job_id=result_data['jobId'], + status_url=result_data['statusUrl'], + execution_id=result_data.get('executionId'), + message=result_data.get('message', ''), + async_execution=result_data.get('async', True) ) return WorkflowExecutionResult( @@ -374,12 +376,12 @@ def close(self) -> None: """Close the underlying HTTP session.""" self._session.close() - def get_job_status(self, task_id: str) -> Dict[str, Any]: + def get_job_status(self, job_id: str) -> Dict[str, Any]: """ Get the status of an async job. Args: - task_id: The task ID returned from async execution + job_id: The job ID returned from async execution Returns: Dictionary containing the job status @@ -387,7 +389,7 @@ def get_job_status(self, task_id: str) -> Dict[str, Any]: Raises: SimStudioError: If getting the status fails """ - url = f"{self.base_url}/api/jobs/{task_id}" + url = f"{self.base_url}/api/jobs/{job_id}" try: response = self._session.get(url) diff --git a/packages/python-sdk/tests/test_client.py b/packages/python-sdk/tests/test_client.py index 8dfdee99b61..814ad7610ef 100644 --- a/packages/python-sdk/tests/test_client.py +++ b/packages/python-sdk/tests/test_client.py @@ -95,17 +95,18 @@ def test_context_manager(mock_close): @patch('simstudio.requests.Session.post') -def test_async_execution_returns_task_id(mock_post): +def test_async_execution_returns_job_id(mock_post): """Test async execution returns AsyncExecutionResult.""" mock_response = Mock() mock_response.ok = True mock_response.status_code = 202 mock_response.json.return_value = { "success": True, - "taskId": "task-123", - "status": "queued", - "createdAt": "2024-01-01T00:00:00Z", - "links": {"status": "/api/jobs/task-123"} + "jobId": "job-123", + "statusUrl": "https://test.sim.ai/api/jobs/job-123", + "executionId": "execution-123", + "message": "Workflow execution started", + "async": True } mock_response.headers.get.return_value = None mock_post.return_value = mock_response @@ -118,9 +119,10 @@ def test_async_execution_returns_task_id(mock_post): ) assert result.success is True - assert result.task_id == "task-123" - assert result.status == "queued" - assert result.links["status"] == "/api/jobs/task-123" + assert result.job_id == "job-123" + assert result.status_url == "https://test.sim.ai/api/jobs/job-123" + assert result.execution_id == "execution-123" + assert result.async_execution is True call_args = mock_post.call_args assert call_args[1]["headers"]["X-Execution-Mode"] == "async" diff --git a/packages/ts-sdk/README.md b/packages/ts-sdk/README.md index 44d21d0c9ed..0ce547f6e51 100644 --- a/packages/ts-sdk/README.md +++ b/packages/ts-sdk/README.md @@ -125,17 +125,17 @@ const result = await client.executeWorkflowSync('workflow-id', { data: 'some inp **Returns:** `Promise` -##### getJobStatus(taskId) +##### getJobStatus(jobId) Get the status of an async job. ```typescript -const status = await client.getJobStatus('task-id-from-async-execution'); +const status = await client.getJobStatus('job-id-from-async-execution'); console.log('Job status:', status); ``` **Parameters:** -- `taskId` (string): The task ID returned from async execution +- `jobId` (string): The job ID returned from async execution **Returns:** `Promise` @@ -226,6 +226,24 @@ interface WorkflowExecutionResult { } ``` +### LargeValueRef + +Oversized execution values may be returned as a versioned reference inside `output`, `logs`, streaming events, or async job status responses. +The `key` field is an opaque execution-scoped server storage pointer, not a client-readable download URL. + +```typescript +interface LargeValueRef { + __simLargeValueRef: true; + version: 1; + id: string; + kind: 'array' | 'object' | 'string' | 'json'; + size: number; + key?: string; + executionId?: string; + preview?: unknown; +} +``` + ### WorkflowStatus ```typescript @@ -250,12 +268,11 @@ class SimStudioError extends Error { ```typescript interface AsyncExecutionResult { success: boolean; - taskId: string; - status: 'queued'; - createdAt: string; - links: { - status: string; - }; + jobId: string; + statusUrl: string; + executionId?: string; + message: string; + async: true; } ``` diff --git a/packages/ts-sdk/src/index.ts b/packages/ts-sdk/src/index.ts index 31f7a34f263..ffed7ca1e7b 100644 --- a/packages/ts-sdk/src/index.ts +++ b/packages/ts-sdk/src/index.ts @@ -5,6 +5,18 @@ export interface SimStudioConfig { baseUrl?: string } +export interface LargeValueRef { + __simLargeValueRef: true + version: 1 + id: string + kind: 'array' | 'object' | 'string' | 'json' + size: number + /** Opaque execution-scoped server storage key. This is not a download URL. */ + key?: string + executionId?: string + preview?: unknown +} + export interface WorkflowExecutionResult { success: boolean output?: any diff --git a/packages/workflow-persistence/src/load.ts b/packages/workflow-persistence/src/load.ts index 3f6f8d2de39..288e9217e8a 100644 --- a/packages/workflow-persistence/src/load.ts +++ b/packages/workflow-persistence/src/load.ts @@ -4,6 +4,7 @@ import type { BlockState, Loop, Parallel } from '@sim/workflow-types/workflow' import { SUBFLOW_TYPES } from '@sim/workflow-types/workflow' import { and, eq, isNull } from 'drizzle-orm' import type { Edge } from 'reactflow' +import { clampParallelBatchSize } from './subflow-helpers' import type { DbOrTx, NormalizedWorkflowData } from './types' const logger = createLogger('WorkflowPersistenceLoad') @@ -141,9 +142,24 @@ export async function loadWorkflowFromNormalizedTablesRaw( (config as Parallel).parallelType === 'collection' ? (config as Parallel).parallelType : 'count', + batchSize: clampParallelBatchSize((config as Parallel).batchSize), enabled: blocksMap[subflow.id]?.enabled ?? true, } parallels[subflow.id] = parallel + + if (blocksMap[subflow.id]) { + const block = blocksMap[subflow.id] + blocksMap[subflow.id] = { + ...block, + data: { + ...block.data, + count: parallel.count, + collection: parallel.distribution ?? block.data?.collection ?? '', + parallelType: parallel.parallelType, + batchSize: parallel.batchSize, + }, + } + } } else { logger.warn(`Unknown subflow type: ${subflow.type} for subflow ${subflow.id}`) } diff --git a/packages/workflow-persistence/src/subflow-helpers.ts b/packages/workflow-persistence/src/subflow-helpers.ts index b0f552f1977..cf0c92b370b 100644 --- a/packages/workflow-persistence/src/subflow-helpers.ts +++ b/packages/workflow-persistence/src/subflow-helpers.ts @@ -1,6 +1,16 @@ import type { BlockState, Loop, Parallel } from '@sim/workflow-types/workflow' const DEFAULT_LOOP_ITERATIONS = 5 +const DEFAULT_PARALLEL_BATCH_SIZE = 20 +const MAX_PARALLEL_BATCH_SIZE = 20 + +export function clampParallelBatchSize(batchSize: unknown): number { + const parsed = typeof batchSize === 'number' ? batchSize : Number.parseInt(String(batchSize), 10) + if (Number.isNaN(parsed)) { + return DEFAULT_PARALLEL_BATCH_SIZE + } + return Math.max(1, Math.min(MAX_PARALLEL_BATCH_SIZE, parsed)) +} export function findChildNodes(containerId: string, blocks: Record): string[] { return Object.values(blocks) @@ -50,6 +60,7 @@ export function convertParallelBlockToParallel( validatedParallelType === 'collection' ? parallelBlock.data?.collection || '' : undefined const count = parallelBlock.data?.count || 5 + const batchSize = clampParallelBatchSize(parallelBlock.data?.batchSize) return { id: parallelBlockId, @@ -57,6 +68,7 @@ export function convertParallelBlockToParallel( distribution, count, parallelType: validatedParallelType, + batchSize, enabled: parallelBlock.enabled, } } diff --git a/packages/workflow-types/src/workflow.ts b/packages/workflow-types/src/workflow.ts index 006bd2ccab6..06b9692ddba 100644 --- a/packages/workflow-types/src/workflow.ts +++ b/packages/workflow-types/src/workflow.ts @@ -25,6 +25,7 @@ export interface ParallelConfig { nodes: string[] distribution?: unknown[] | Record | string parallelType?: 'count' | 'collection' + batchSize?: number } export interface Subflow { @@ -52,6 +53,7 @@ export interface BlockData { whileCondition?: string doWhileCondition?: string parallelType?: 'collection' | 'count' + batchSize?: number type?: string canonicalModes?: Record } @@ -178,6 +180,7 @@ export interface Parallel { distribution?: any[] | Record | string count?: number parallelType?: 'count' | 'collection' + batchSize?: number enabled: boolean locked?: boolean }