From df581c3efb745802ba65f37af8fa8e71f1a0687a Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Fri, 24 Apr 2026 18:31:46 -0700 Subject: [PATCH 1/2] fix(mothership): queue supersede crash (#4297) * fix(mothership): queue supersede crash * add test * abort observed marker --- .../sim/lib/copilot/request/go/stream.test.ts | 144 +++++++++++++++++- apps/sim/lib/copilot/request/go/stream.ts | 38 +++-- .../lib/copilot/request/lifecycle/headless.ts | 8 +- .../copilot/request/lifecycle/start.test.ts | 46 +++++- .../lib/copilot/request/lifecycle/start.ts | 7 +- .../copilot/request/session/abort-reason.ts | 12 +- .../lib/copilot/request/session/abort.test.ts | 41 +++++ apps/sim/lib/copilot/request/session/abort.ts | 2 +- apps/sim/lib/copilot/request/types.ts | 8 + 9 files changed, 287 insertions(+), 19 deletions(-) diff --git a/apps/sim/lib/copilot/request/go/stream.test.ts b/apps/sim/lib/copilot/request/go/stream.test.ts index 8234658ddcc..8d08f755d8f 100644 --- a/apps/sim/lib/copilot/request/go/stream.test.ts +++ b/apps/sim/lib/copilot/request/go/stream.test.ts @@ -10,13 +10,24 @@ import { MothershipStreamV1ToolOutcome, MothershipStreamV1ToolPhase, } from '@/lib/copilot/generated/mothership-stream-v1' + +vi.mock('@/lib/copilot/request/session', async () => { + const actual = await vi.importActual( + '@/lib/copilot/request/session' + ) + return { + ...actual, + hasAbortMarker: vi.fn().mockResolvedValue(false), + } +}) + import { buildPreviewContentUpdate, decodeJsonStringPrefix, extractEditContent, runStreamLoop, } from '@/lib/copilot/request/go/stream' -import { createEvent } from '@/lib/copilot/request/session' +import { AbortReason, createEvent, hasAbortMarker } from '@/lib/copilot/request/session' import { RequestTraceV1Outcome, TraceCollector } from '@/lib/copilot/request/trace' import type { ExecutionContext, StreamingContext } from '@/lib/copilot/request/types' @@ -285,6 +296,137 @@ describe('copilot go stream helpers', () => { ).toBe(true) }) + it('reclassifies as aborted when the body closes without terminal but the abort marker is set', async () => { + const textEvent = createEvent({ + streamId: 'stream-1', + cursor: '1', + seq: 1, + requestId: 'req-1', + type: MothershipStreamV1EventType.text, + payload: { + channel: 'assistant', + text: 'partial response', + }, + }) + + vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent])) + vi.mocked(hasAbortMarker).mockResolvedValueOnce(true) + + const context = createStreamingContext() + const execContext: ExecutionContext = { + userId: 'user-1', + workflowId: 'workflow-1', + } + + await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, { + timeout: 1000, + }) + + expect(hasAbortMarker).toHaveBeenCalledWith(context.messageId) + expect(context.wasAborted).toBe(true) + expect( + context.errors.some((message) => + message.includes('Copilot backend stream ended before a terminal event') + ) + ).toBe(false) + }) + + it('invokes onAbortObserved with MarkerObservedAtBodyClose when reclassifying via the abort marker', async () => { + const textEvent = createEvent({ + streamId: 'stream-1', + cursor: '1', + seq: 1, + requestId: 'req-1', + type: MothershipStreamV1EventType.text, + payload: { + channel: 'assistant', + text: 'partial response', + }, + }) + + vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent])) + vi.mocked(hasAbortMarker).mockResolvedValueOnce(true) + + const context = createStreamingContext() + const execContext: ExecutionContext = { + userId: 'user-1', + workflowId: 'workflow-1', + } + const onAbortObserved = vi.fn() + + await runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, { + timeout: 1000, + onAbortObserved, + }) + + expect(onAbortObserved).toHaveBeenCalledTimes(1) + expect(onAbortObserved).toHaveBeenCalledWith(AbortReason.MarkerObservedAtBodyClose) + expect(context.wasAborted).toBe(true) + }) + + it('does not invoke onAbortObserved when no abort marker is present at body close', async () => { + const textEvent = createEvent({ + streamId: 'stream-1', + cursor: '1', + seq: 1, + requestId: 'req-1', + type: MothershipStreamV1EventType.text, + payload: { + channel: 'assistant', + text: 'partial response', + }, + }) + + vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent])) + vi.mocked(hasAbortMarker).mockResolvedValueOnce(false) + + const context = createStreamingContext() + const execContext: ExecutionContext = { + userId: 'user-1', + workflowId: 'workflow-1', + } + const onAbortObserved = vi.fn() + + await expect( + runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, { + timeout: 1000, + onAbortObserved, + }) + ).rejects.toThrow('Copilot backend stream ended before a terminal event') + + expect(onAbortObserved).not.toHaveBeenCalled() + }) + + it('still fails closed when the body closes without terminal and the abort marker check throws', async () => { + const textEvent = createEvent({ + streamId: 'stream-1', + cursor: '1', + seq: 1, + requestId: 'req-1', + type: MothershipStreamV1EventType.text, + payload: { + channel: 'assistant', + text: 'partial response', + }, + }) + + vi.mocked(fetch).mockResolvedValueOnce(createSseResponse([textEvent])) + vi.mocked(hasAbortMarker).mockRejectedValueOnce(new Error('redis unavailable')) + + const context = createStreamingContext() + const execContext: ExecutionContext = { + userId: 'user-1', + workflowId: 'workflow-1', + } + + await expect( + runStreamLoop('https://example.com/mothership/stream', {}, context, execContext, { + timeout: 1000, + }) + ).rejects.toThrow('Copilot backend stream ended before a terminal event') + expect(context.wasAborted).toBe(false) + }) + it('fails closed when the shared stream receives an invalid event', async () => { vi.mocked(fetch).mockResolvedValueOnce( createSseResponse([ diff --git a/apps/sim/lib/copilot/request/go/stream.ts b/apps/sim/lib/copilot/request/go/stream.ts index c92d135affc..ddbf7678d6b 100644 --- a/apps/sim/lib/copilot/request/go/stream.ts +++ b/apps/sim/lib/copilot/request/go/stream.ts @@ -30,7 +30,9 @@ import { } from '@/lib/copilot/request/handlers/types' import { getCopilotTracer } from '@/lib/copilot/request/otel' import { + AbortReason, eventToStreamEvent, + hasAbortMarker, isSubagentSpanStreamEvent, parsePersistedStreamEventEnvelope, } from '@/lib/copilot/request/session' @@ -436,16 +438,32 @@ export async function runStreamLoop( }) if (!context.streamComplete && !abortSignal?.aborted && !context.wasAborted) { - const streamPath = new URL(fetchUrl).pathname - const message = `Copilot backend stream ended before a terminal event on ${streamPath}` - context.errors.push(message) - logger.error('Copilot backend stream ended before a terminal event', { - path: streamPath, - requestId: context.requestId, - messageId: context.messageId, - }) - endedOn = CopilotSseCloseReason.ClosedNoTerminal - throw new CopilotBackendError(message, { status: 503 }) + let abortRequested = false + try { + abortRequested = await hasAbortMarker(context.messageId) + } catch (error) { + logger.warn('Failed to read abort marker at body close', { + streamId: context.messageId, + error: error instanceof Error ? error.message : String(error), + }) + } + + if (abortRequested) { + options.onAbortObserved?.(AbortReason.MarkerObservedAtBodyClose) + context.wasAborted = true + endedOn = CopilotSseCloseReason.Aborted + } else { + const streamPath = new URL(fetchUrl).pathname + const message = `Copilot backend stream ended before a terminal event on ${streamPath}` + context.errors.push(message) + logger.error('Copilot backend stream ended before a terminal event', { + path: streamPath, + requestId: context.requestId, + messageId: context.messageId, + }) + endedOn = CopilotSseCloseReason.ClosedNoTerminal + throw new CopilotBackendError(message, { status: 503 }) + } } } catch (error) { if (error instanceof FatalSseEventError && !context.errors.includes(error.message)) { diff --git a/apps/sim/lib/copilot/request/lifecycle/headless.ts b/apps/sim/lib/copilot/request/lifecycle/headless.ts index 69527829316..381e91a3953 100644 --- a/apps/sim/lib/copilot/request/lifecycle/headless.ts +++ b/apps/sim/lib/copilot/request/lifecycle/headless.ts @@ -53,10 +53,10 @@ export async function runHeadlessCopilotLifecycle( simRequestId, otelContext, }) - outcome = options.abortSignal?.aborted - ? RequestTraceV1Outcome.cancelled - : result.success - ? RequestTraceV1Outcome.success + outcome = result.success + ? RequestTraceV1Outcome.success + : options.abortSignal?.aborted || result.cancelled + ? RequestTraceV1Outcome.cancelled : RequestTraceV1Outcome.error return result } catch (error) { diff --git a/apps/sim/lib/copilot/request/lifecycle/start.test.ts b/apps/sim/lib/copilot/request/lifecycle/start.test.ts index 5477fc9994b..e27f2a2919f 100644 --- a/apps/sim/lib/copilot/request/lifecycle/start.test.ts +++ b/apps/sim/lib/copilot/request/lifecycle/start.test.ts @@ -6,7 +6,10 @@ import { propagation, trace } from '@opentelemetry/api' import { W3CTraceContextPropagator } from '@opentelemetry/core' import { BasicTracerProvider } from '@opentelemetry/sdk-trace-base' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { MothershipStreamV1EventType } from '@/lib/copilot/generated/mothership-stream-v1' +import { + MothershipStreamV1CompletionStatus, + MothershipStreamV1EventType, +} from '@/lib/copilot/generated/mothership-stream-v1' const { runCopilotLifecycle, @@ -60,6 +63,7 @@ vi.mock('@/lib/copilot/request/session', () => ({ registerActiveStream: vi.fn(), unregisterActiveStream: vi.fn(), startAbortPoller: vi.fn().mockReturnValue(setInterval(() => {}, 999999)), + isExplicitStopReason: vi.fn().mockReturnValue(false), SSE_RESPONSE_HEADERS: {}, StreamWriter: vi.fn().mockImplementation(() => ({ attach: vi.fn().mockImplementation((ctrl: ReadableStreamDefaultController) => { @@ -211,6 +215,46 @@ describe('createSSEStream terminal error handling', () => { expect(scheduleBufferCleanup).toHaveBeenCalledWith('stream-1') }) + it('publishes a cancelled completion (not an error) when the orchestrator reports cancelled without abortSignal aborted', async () => { + runCopilotLifecycle.mockResolvedValue({ + success: false, + cancelled: true, + content: '', + contentBlocks: [], + toolCalls: [], + }) + + const stream = createSSEStream({ + requestPayload: { message: 'hello' }, + userId: 'user-1', + streamId: 'stream-1', + executionId: 'exec-1', + runId: 'run-1', + currentChat: null, + isNewChat: false, + message: 'hello', + titleModel: 'gpt-5.4', + requestId: 'req-cancelled', + orchestrateOptions: {}, + }) + + await drainStream(stream) + + expect(appendEvent).not.toHaveBeenCalledWith( + expect.objectContaining({ + type: MothershipStreamV1EventType.error, + }) + ) + expect(appendEvent).toHaveBeenCalledWith( + expect.objectContaining({ + type: MothershipStreamV1EventType.complete, + payload: expect.objectContaining({ + status: MothershipStreamV1CompletionStatus.cancelled, + }), + }) + ) + }) + it('passes an OTel context into the streaming lifecycle', async () => { let lifecycleTraceparent = '' runCopilotLifecycle.mockImplementation(async (_payload, options) => { diff --git a/apps/sim/lib/copilot/request/lifecycle/start.ts b/apps/sim/lib/copilot/request/lifecycle/start.ts index 37d58624c17..d401855b9c1 100644 --- a/apps/sim/lib/copilot/request/lifecycle/start.ts +++ b/apps/sim/lib/copilot/request/lifecycle/start.ts @@ -249,6 +249,11 @@ export function createSSEStream(params: StreamingOrchestrationParams): ReadableS onEvent: async (event) => { await publisher.publish(event) }, + onAbortObserved: (reason) => { + if (!abortController.signal.aborted) { + abortController.abort(reason) + } + }, }) lifecycleResult = result @@ -266,7 +271,7 @@ export function createSSEStream(params: StreamingOrchestrationParams): ReadableS // 3. Otherwise → error. outcome = result.success ? RequestTraceV1Outcome.success - : abortController.signal.aborted || publisher.clientDisconnected + : result.cancelled || abortController.signal.aborted || publisher.clientDisconnected ? RequestTraceV1Outcome.cancelled : RequestTraceV1Outcome.error if (outcome === RequestTraceV1Outcome.cancelled) { diff --git a/apps/sim/lib/copilot/request/session/abort-reason.ts b/apps/sim/lib/copilot/request/session/abort-reason.ts index fa850d63f15..8a6b281e2c0 100644 --- a/apps/sim/lib/copilot/request/session/abort-reason.ts +++ b/apps/sim/lib/copilot/request/session/abort-reason.ts @@ -22,6 +22,12 @@ export const AbortReason = { * that the node that DID receive it wrote, and aborts on the poll. */ RedisPoller: 'redis_abort_marker:poller', + /** + * Cross-process stop: same root cause as `RedisPoller`, but observed + * by `runStreamLoop` at body close (the Go body ended before the + * 250ms poller's next tick) rather than by the polling timer. + */ + MarkerObservedAtBodyClose: 'redis_abort_marker:body_close', /** Internal timeout on the outbound explicit-abort fetch to Go. */ ExplicitAbortFetchTimeout: 'timeout:go_explicit_abort_fetch', } as const @@ -38,5 +44,9 @@ export type AbortReasonValue = (typeof AbortReason)[keyof typeof AbortReason] * stops, mirroring `requestctx.IsExplicitUserStop` on the Go side. */ export function isExplicitStopReason(reason: unknown): boolean { - return reason === AbortReason.UserStop || reason === AbortReason.RedisPoller + return ( + reason === AbortReason.UserStop || + reason === AbortReason.RedisPoller || + reason === AbortReason.MarkerObservedAtBodyClose + ) } diff --git a/apps/sim/lib/copilot/request/session/abort.test.ts b/apps/sim/lib/copilot/request/session/abort.test.ts index 9c6ee82aa08..bdfd5d39cbb 100644 --- a/apps/sim/lib/copilot/request/session/abort.test.ts +++ b/apps/sim/lib/copilot/request/session/abort.test.ts @@ -98,6 +98,47 @@ describe('startAbortPoller heartbeat', () => { } }) + it('aborts the controller before clearing the marker so the marker is never observable as cleared while the signal is still unaborted', async () => { + const controller = new AbortController() + const streamId = 'stream-order-1' + + let signalAbortedWhenMarkerCleared: boolean | null = null + mockClearAbortMarker.mockImplementationOnce(async () => { + signalAbortedWhenMarkerCleared = controller.signal.aborted + }) + mockHasAbortMarker.mockResolvedValueOnce(true) + + const interval = startAbortPoller(streamId, controller, {}) + + try { + await vi.advanceTimersByTimeAsync(300) + + expect(mockClearAbortMarker).toHaveBeenCalledWith(streamId) + expect(signalAbortedWhenMarkerCleared).toBe(true) + expect(controller.signal.aborted).toBe(true) + } finally { + clearInterval(interval) + } + }) + + it('does not clear the marker when the signal is already aborted (no double abort)', async () => { + const controller = new AbortController() + controller.abort('preexisting') + const streamId = 'stream-order-2' + + mockHasAbortMarker.mockResolvedValueOnce(true) + + const interval = startAbortPoller(streamId, controller, {}) + + try { + await vi.advanceTimersByTimeAsync(300) + + expect(mockClearAbortMarker).not.toHaveBeenCalled() + } finally { + clearInterval(interval) + } + }) + it('stops heartbeating after ownership is lost', async () => { const controller = new AbortController() const streamId = 'stream-lost' diff --git a/apps/sim/lib/copilot/request/session/abort.ts b/apps/sim/lib/copilot/request/session/abort.ts index db3beff57ea..ce508690361 100644 --- a/apps/sim/lib/copilot/request/session/abort.ts +++ b/apps/sim/lib/copilot/request/session/abort.ts @@ -17,7 +17,7 @@ const pendingChatStreams = new Map< { promise: Promise; resolve: () => void; streamId: string } >() -const DEFAULT_ABORT_POLL_MS = 1000 +const DEFAULT_ABORT_POLL_MS = 250 /** * TTL for the per-chat stream lock. Kept short so that if the Sim pod diff --git a/apps/sim/lib/copilot/request/types.ts b/apps/sim/lib/copilot/request/types.ts index bbb6264fd82..d2fff5be35a 100644 --- a/apps/sim/lib/copilot/request/types.ts +++ b/apps/sim/lib/copilot/request/types.ts @@ -136,6 +136,14 @@ export interface OrchestratorOptions { onComplete?: (result: OrchestratorResult) => void | Promise onError?: (error: Error) => void | Promise abortSignal?: AbortSignal + /** + * Invoked when the orchestrator infers that the run was aborted via + * an out-of-band signal (currently: a Redis abort marker observed + * at SSE body close). Callers wire this to fire their local + * `AbortController` so `signal.reason` is set and `recordCancelled` + * classifies as `explicit_stop` rather than `unknown`. + */ + onAbortObserved?: (reason: string) => void interactive?: boolean } From d93a6f57bc78dac6598ae61d5497f94febbe44e5 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Fri, 24 Apr 2026 18:47:57 -0700 Subject: [PATCH 2/2] chore(guide): update contributing guide (#4296) * chore(guide): update contributing guide * fix md --- .github/CONTRIBUTING.md | 259 +++++++++++++++++++++++++--------------- 1 file changed, 166 insertions(+), 93 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 40d1324323e..f4e3df0d31c 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -2,8 +2,15 @@ Thank you for your interest in contributing to Sim! Our goal is to provide developers with a powerful, user-friendly platform for building, testing, and optimizing agentic workflows. We welcome contributions in all forms—from bug fixes and design improvements to brand-new features. -> **Project Overview:** -> Sim is a monorepo using Turborepo, containing the main application (`apps/sim/`), documentation (`apps/docs/`), and shared packages (`packages/`). The main application is built with Next.js (app router), ReactFlow, Zustand, Shadcn, and Tailwind CSS. Please ensure your contributions follow our best practices for clarity, maintainability, and consistency. +> **Project Overview:** +> Sim is a Turborepo monorepo with two deployable apps and a set of shared packages: +> +> - `apps/sim/` — the main Next.js application (App Router, ReactFlow, Zustand, Shadcn, Tailwind CSS). +> - `apps/realtime/` — a small Bun + Socket.IO server that powers the collaborative canvas. Shares DB and Better Auth secrets with `apps/sim` via `@sim/*` packages. +> - `apps/docs/` — Fumadocs-based documentation site. +> - `packages/` — shared workspace packages (`@sim/db`, `@sim/auth`, `@sim/audit`, `@sim/workflow-types`, `@sim/workflow-persistence`, `@sim/workflow-authz`, `@sim/realtime-protocol`, `@sim/security`, `@sim/logger`, `@sim/utils`, `@sim/testing`, `@sim/tsconfig`). +> +> Strict one-way dependency flow: `apps/* → packages/*`. Packages never import from apps. Please ensure your contributions follow this and our best practices for clarity, maintainability, and consistency. --- @@ -24,14 +31,17 @@ Thank you for your interest in contributing to Sim! Our goal is to provide devel We strive to keep our workflow as simple as possible. To contribute: -1. **Fork the Repository** +1. **Fork the Repository** Click the **Fork** button on GitHub to create your own copy of the project. 2. **Clone Your Fork** + ```bash git clone https://github.com//sim.git + cd sim ``` -3. **Create a Feature Branch** + +3. **Create a Feature Branch** Create a new branch with a descriptive name: ```bash @@ -40,21 +50,23 @@ We strive to keep our workflow as simple as possible. To contribute: Use a clear naming convention to indicate the type of work (e.g., `feat/`, `fix/`, `docs/`). -4. **Make Your Changes** +4. **Make Your Changes** Ensure your changes are small, focused, and adhere to our coding guidelines. -5. **Commit Your Changes** +5. **Commit Your Changes** Write clear, descriptive commit messages that follow the [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/#specification) specification. This allows us to maintain a coherent project history and generate changelogs automatically. For example: + - `feat(api): add new endpoint for user authentication` - `fix(ui): resolve button alignment issue` - `docs: update contribution guidelines` + 6. **Push Your Branch** ```bash git push origin feat/your-feature-name ``` -7. **Create a Pull Request** +7. **Create a Pull Request** Open a pull request against the `staging` branch on GitHub. Please provide a clear description of the changes and reference any relevant issues (e.g., `fixes #123`). --- @@ -65,7 +77,7 @@ If you discover a bug or have a feature request, please open an issue in our Git - Provide a clear, descriptive title. - Include as many details as possible (steps to reproduce, screenshots, etc.). -- **Tag Your Issue Appropriately:** +- **Tag Your Issue Appropriately:** Use the following labels to help us categorize your issue: - **active:** Actively working on it right now. - **bug:** Something isn't working. @@ -82,12 +94,11 @@ If you discover a bug or have a feature request, please open an issue in our Git Before creating a pull request: -- **Ensure Your Branch Is Up-to-Date:** +- **Ensure Your Branch Is Up-to-Date:** Rebase your branch onto the latest `staging` branch to prevent merge conflicts. -- **Follow the Guidelines:** +- **Follow the Guidelines:** Make sure your changes are well-tested, follow our coding standards, and include relevant documentation if necessary. - -- **Reference Issues:** +- **Reference Issues:** If your PR addresses an existing issue, include `refs #` or `fixes #` in your PR description. Our maintainers will review your pull request and provide feedback. We aim to make the review process as smooth and timely as possible. @@ -166,27 +177,27 @@ To use local models with Sim: 1. Install Ollama and pull models: -```bash -# Install Ollama (if not already installed) -curl -fsSL https://ollama.ai/install.sh | sh + ```bash + # Install Ollama (if not already installed) + curl -fsSL https://ollama.ai/install.sh | sh -# Pull a model (e.g., gemma3:4b) -ollama pull gemma3:4b -``` + # Pull a model (e.g., gemma3:4b) + ollama pull gemma3:4b + ``` 2. Start Sim with local model support: -```bash -# With NVIDIA GPU support -docker compose --profile local-gpu -f docker-compose.ollama.yml up -d + ```bash + # With NVIDIA GPU support + docker compose --profile local-gpu -f docker-compose.ollama.yml up -d -# Without GPU (CPU only) -docker compose --profile local-cpu -f docker-compose.ollama.yml up -d + # Without GPU (CPU only) + docker compose --profile local-cpu -f docker-compose.ollama.yml up -d -# If hosting on a server, update the environment variables in the docker-compose.prod.yml file -# to include the server's public IP then start again (OLLAMA_URL to i.e. http://1.1.1.1:11434) -docker compose -f docker-compose.prod.yml up -d -``` + # If hosting on a server, update the environment variables in the docker-compose.prod.yml file + # to include the server's public IP then start again (OLLAMA_URL to i.e. http://1.1.1.1:11434) + docker compose -f docker-compose.prod.yml up -d + ``` ### Option 3: Using VS Code / Cursor Dev Containers @@ -201,61 +212,104 @@ Dev Containers provide a consistent and easy-to-use development environment: 2. **Setup Steps:** - Clone the repository: + ```bash git clone https://github.com//sim.git cd sim ``` - - Open the project in VS Code/Cursor - - When prompted, click "Reopen in Container" (or press F1 and select "Remote-Containers: Reopen in Container") - - Wait for the container to build and initialize + + - Open the project in VS Code/Cursor. + - When prompted, click "Reopen in Container" (or press F1 and select "Remote-Containers: Reopen in Container"). + - Wait for the container to build and initialize. 3. **Start Developing:** - - Run `bun run dev:full` in the terminal or use the `sim-start` alias - - This starts both the main application and the realtime socket server - - All dependencies and configurations are automatically set up - - Your changes will be automatically hot-reloaded + - Run `bun run dev:full` in the terminal or use the `sim-start` alias. + - This starts both the main application and the realtime socket server. + - All dependencies and configurations are automatically set up. + - Your changes will be automatically hot-reloaded. 4. **GitHub Codespaces:** - - This setup also works with GitHub Codespaces if you prefer development in the browser - - Just click "Code" → "Codespaces" → "Create codespace on staging" + + - This setup also works with GitHub Codespaces if you prefer development in the browser. + - Just click "Code" → "Codespaces" → "Create codespace on staging". ### Option 4: Manual Setup -If you prefer not to use Docker or Dev Containers: +If you prefer not to use Docker or Dev Containers. **All commands run from the repository root unless explicitly noted.** + +1. **Clone and Install:** -1. **Clone the Repository:** ```bash git clone https://github.com//sim.git cd sim bun install ``` -2. **Set Up Environment:** + Bun workspaces handle dependency resolution for all apps and packages from the root `bun install`. - - Navigate to the app directory: - ```bash - cd apps/sim - ``` - - Copy `.env.example` to `.env` - - Configure required variables (DATABASE_URL, BETTER_AUTH_SECRET, BETTER_AUTH_URL) +2. **Set Up Environment Files:** + + We use **per-app `.env` files** (the Turborepo-canonical pattern), not a single root `.env`. Three files are needed for local dev: + + ```bash + # Main app — large, app-specific (OAuth secrets, LLM keys, Stripe, etc.) + cp apps/sim/.env.example apps/sim/.env + + # Realtime server — small, only the values shared with the main app + cp apps/realtime/.env.example apps/realtime/.env + + # DB tooling (drizzle-kit, db:migrate) + cp packages/db/.env.example packages/db/.env + ``` + + At minimum, each `.env` needs `DATABASE_URL`. `apps/sim/.env` and `apps/realtime/.env` additionally need matching values for `BETTER_AUTH_URL`, `BETTER_AUTH_SECRET`, `INTERNAL_API_SECRET`, and `NEXT_PUBLIC_APP_URL`. `apps/sim/.env` also needs `ENCRYPTION_KEY` and `API_ENCRYPTION_KEY`. Generate any 32-char secrets with `openssl rand -hex 32`. + + The same `BETTER_AUTH_SECRET`, `INTERNAL_API_SECRET`, and `DATABASE_URL` must appear in both `apps/sim/.env` and `apps/realtime/.env` so the two services share auth and DB. After editing `apps/sim/.env`, you can mirror the shared subset into the realtime env in one shot: -3. **Set Up Database:** + ```bash + grep -E '^(DATABASE_URL|BETTER_AUTH_URL|BETTER_AUTH_SECRET|INTERNAL_API_SECRET|NEXT_PUBLIC_APP_URL|REDIS_URL)=' apps/sim/.env > apps/realtime/.env + grep -E '^DATABASE_URL=' apps/sim/.env > packages/db/.env + ``` + +3. **Run Database Migrations:** + + Migrations live in `packages/db/migrations/`. Run them via the dedicated workspace script: ```bash - bunx drizzle-kit push + cd packages/db && bun run db:migrate && cd ../.. ``` -4. **Run the Development Server:** + For ad-hoc schema iteration during development you can also use `bun run db:push` from `packages/db`, but `db:migrate` is the canonical command for both local and CI/CD setups. + +4. **Run the Development Servers:** ```bash bun run dev:full ``` - This command starts both the main application and the realtime socket server required for full functionality. + This launches both apps with coloured prefixes: + + - `[App]` — Next.js on `http://localhost:3000` + - `[Realtime]` — Socket.IO on `http://localhost:3002` + + Or run them separately: + + ```bash + bun run dev # Next.js app only + bun run dev:sockets # realtime server only + ``` 5. **Make Your Changes and Test Locally.** + Before opening a PR, run the same checks CI runs: + + ```bash + bun run type-check # TypeScript across every workspace + bun run lint:check # Biome lint across every workspace + bun run test # Vitest across every workspace + ``` + ### Email Template Development When working on email templates, you can preview them using a local email preview server: @@ -263,18 +317,19 @@ When working on email templates, you can preview them using a local email previe 1. **Run the Email Preview Server:** ```bash - bun run email:dev + cd apps/sim && bun run email:dev ``` 2. **Access the Preview:** - - Open `http://localhost:3000` in your browser - - You'll see a list of all email templates - - Click on any template to view and test it with various parameters + - Open `http://localhost:3000` in your browser. + - You'll see a list of all email templates. + - Click on any template to view and test it with various parameters. 3. **Templates Location:** - - Email templates are located in `sim/app/emails/` - - After making changes to templates, they will automatically update in the preview + + - Email templates live in `apps/sim/components/emails/`. + - Changes hot-reload automatically in the preview. --- @@ -282,28 +337,41 @@ When working on email templates, you can preview them using a local email previe Sim is built in a modular fashion where blocks and tools extend the platform's functionality. To maintain consistency and quality, please follow the guidelines below when adding a new block or tool. +> **Use the skill guides for step-by-step recipes.** The repository ships opinionated, end-to-end guides under `.agents/skills/` that cover the exact file layout, conventions, registry wiring, and gotchas for each kind of contribution. Read the relevant SKILL.md before you start writing code: +> +> | Adding… | Read | +> | ------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------- | +> | A new integration end-to-end (tools + block + icon + optional triggers + all registrations) | [`.agents/skills/add-integration/SKILL.md`](../.agents/skills/add-integration/SKILL.md) | +> | Just a block (or aligning an existing block with its tools) | [`.agents/skills/add-block/SKILL.md`](../.agents/skills/add-block/SKILL.md) | +> | Just tool configs for a service | [`.agents/skills/add-tools/SKILL.md`](../.agents/skills/add-tools/SKILL.md) | +> | A webhook trigger for a service | [`.agents/skills/add-trigger/SKILL.md`](../.agents/skills/add-trigger/SKILL.md) | +> | A knowledge-base connector (sync docs from an external source) | [`.agents/skills/add-connector/SKILL.md`](../.agents/skills/add-connector/SKILL.md) | +> +> The shorter overview below is a high-level reference; the SKILL.md files are the authoritative source of truth and stay in sync with the codebase. + ### Where to Add Your Code -- **Blocks:** Create your new block file under the `/apps/sim/blocks/blocks` directory. The name of the file should match the provider name (e.g., `pinecone.ts`). -- **Tools:** Create a new directory under `/apps/sim/tools` with the same name as the provider (e.g., `/apps/sim/tools/pinecone`). +- **Blocks:** Create your new block file under the `apps/sim/blocks/blocks/` directory. The name of the file should match the provider name (e.g., `pinecone.ts`). +- **Tools:** Create a new directory under `apps/sim/tools/` with the same name as the provider (e.g., `apps/sim/tools/pinecone`). In addition, you will need to update the registries: -- **Block Registry:** Update the blocks index (`/apps/sim/blocks/index.ts`) to include your new block. -- **Tool Registry:** Update the tools registry (`/apps/sim/tools/index.ts`) to add your new tool. +- **Block Registry:** Add your block to `apps/sim/blocks/registry.ts`. (`apps/sim/blocks/index.ts` re-exports lookups from the registry; you do not need to edit it.) +- **Tool Registry:** Add your tool to `apps/sim/tools/index.ts`. ### How to Create a New Block -1. **Create a New File:** - Create a file for your block named after the provider (e.g., `pinecone.ts`) in the `/apps/sim/blocks/blocks` directory. +1. **Create a New File:** + Create a file for your block named after the provider (e.g., `pinecone.ts`) in the `apps/sim/blocks/blocks/` directory. 2. **Create a New Icon:** - Create a new icon for your block in the `/apps/sim/components/icons.tsx` file. The icon should follow the same naming convention as the block (e.g., `PineconeIcon`). + Create a new icon for your block in `apps/sim/components/icons.tsx`. The icon should follow the same naming convention as the block (e.g., `PineconeIcon`). -3. **Define the Block Configuration:** +3. **Define the Block Configuration:** Your block should export a constant of type `BlockConfig`. For example: - ```typescript:/apps/sim/blocks/blocks/pinecone.ts + ```typescript + // apps/sim/blocks/blocks/pinecone.ts import { PineconeIcon } from '@/components/icons' import type { BlockConfig } from '@/blocks/types' import type { PineconeResponse } from '@/tools/pinecone/types' @@ -321,7 +389,7 @@ In addition, you will need to update the registries: { id: 'operation', title: 'Operation', - type: 'dropdown' + type: 'dropdown', required: true, options: [ { label: 'Generate Embeddings', id: 'generate' }, @@ -332,7 +400,7 @@ In addition, you will need to update the registries: { id: 'apiKey', title: 'API Key', - type: 'short-input' + type: 'short-input', placeholder: 'Your Pinecone API key', password: true, required: true, @@ -370,10 +438,11 @@ In addition, you will need to update the registries: } ``` -4. **Register Your Block:** - Add your block to the blocks registry (`/apps/sim/blocks/registry.ts`): +4. **Register Your Block:** + Add your block to the blocks registry (`apps/sim/blocks/registry.ts`): - ```typescript:/apps/sim/blocks/registry.ts + ```typescript + // apps/sim/blocks/registry.ts import { PineconeBlock } from '@/blocks/blocks/pinecone' // Registry of all available blocks @@ -385,24 +454,25 @@ In addition, you will need to update the registries: The block will be automatically available to the application through the registry. -5. **Test Your Block:** +5. **Test Your Block:** Ensure that the block displays correctly in the UI and that its functionality works as expected. ### How to Create a New Tool -1. **Create a New Directory:** - Create a directory under `/apps/sim/tools` with the same name as the provider (e.g., `/apps/sim/tools/pinecone`). +1. **Create a New Directory:** + Create a directory under `apps/sim/tools/` with the same name as the provider (e.g., `apps/sim/tools/pinecone`). -2. **Create Tool Files:** +2. **Create Tool Files:** Create separate files for each tool functionality with descriptive names (e.g., `fetch.ts`, `generate_embeddings.ts`, `search_text.ts`) in your tool directory. -3. **Create a Types File:** +3. **Create a Types File:** Create a `types.ts` file in your tool directory to define and export all types related to your tools. -4. **Create an Index File:** +4. **Create an Index File:** Create an `index.ts` file in your tool directory that imports and exports all tools: - ```typescript:/apps/sim/tools/pinecone/index.ts + ```typescript + // apps/sim/tools/pinecone/index.ts import { fetchTool } from './fetch' import { generateEmbeddingsTool } from './generate_embeddings' import { searchTextTool } from './search_text' @@ -410,10 +480,11 @@ In addition, you will need to update the registries: export { fetchTool, generateEmbeddingsTool, searchTextTool } ``` -5. **Define the Tool Configuration:** +5. **Define the Tool Configuration:** Your tool should export a constant with a naming convention of `{toolName}Tool`. The tool ID should follow the format `{provider}_{tool_name}`. For example: - ```typescript:/apps/sim/tools/pinecone/fetch.ts + ```typescript + // apps/sim/tools/pinecone/fetch.ts import { ToolConfig, ToolResponse } from '@/tools/types' import { PineconeParams, PineconeResponse } from '@/tools/pinecone/types' @@ -449,11 +520,12 @@ In addition, you will need to update the registries: } ``` -6. **Register Your Tool:** - Update the tools registry in `/apps/sim/tools/index.ts` to include your new tool: +6. **Register Your Tool:** + Update the tools registry in `apps/sim/tools/index.ts` to include your new tool: - ```typescript:/apps/sim/tools/index.ts - import { fetchTool, generateEmbeddingsTool, searchTextTool } from '/@tools/pinecone' + ```typescript + // apps/sim/tools/index.ts + import { fetchTool, generateEmbeddingsTool, searchTextTool } from '@/tools/pinecone' // ... other imports export const tools: Record = { @@ -464,13 +536,14 @@ In addition, you will need to update the registries: } ``` -7. **Test Your Tool:** +7. **Test Your Tool:** Ensure that your tool functions correctly by making test requests and verifying the responses. -8. **Generate Documentation:** - Run the documentation generator to create docs for your new tool: +8. **Generate Documentation:** + Run the documentation generator (from `apps/sim`) to create docs for your new tool: + ```bash - ./scripts/generate-docs.sh + cd apps/sim && bun run generate-docs ``` ### Naming Conventions @@ -480,7 +553,7 @@ Maintaining consistent naming across the codebase is critical for auto-generatio - **Block Files:** Name should match the provider (e.g., `pinecone.ts`) - **Block Export:** Should be named `{Provider}Block` (e.g., `PineconeBlock`) - **Icons:** Should be named `{Provider}Icon` (e.g., `PineconeIcon`) -- **Tool Directories:** Should match the provider name (e.g., `/tools/pinecone/`) +- **Tool Directories:** Should match the provider name (e.g., `tools/pinecone/`) - **Tool Files:** Should be named after their function (e.g., `fetch.ts`, `search_text.ts`) - **Tool Exports:** Should be named `{toolName}Tool` (e.g., `fetchTool`) - **Tool IDs:** Should follow the format `{provider}_{tool_name}` (e.g., `pinecone_fetch`) @@ -489,12 +562,12 @@ Maintaining consistent naming across the codebase is critical for auto-generatio Sim implements a sophisticated parameter visibility system that controls how parameters are exposed to users and LLMs in agent workflows. Each parameter can have one of four visibility levels: -| Visibility | User Sees | LLM Sees | How It Gets Set | -|-------------|-----------|----------|--------------------------------| -| `user-only` | ✅ Yes | ❌ No | User provides in UI | -| `user-or-llm` | ✅ Yes | ✅ Yes | User provides OR LLM generates | -| `llm-only` | ❌ No | ✅ Yes | LLM generates only | -| `hidden` | ❌ No | ❌ No | Application injects at runtime | +| Visibility | User Sees | LLM Sees | How It Gets Set | +| ------------- | --------- | -------- | ------------------------------ | +| `user-only` | ✅ Yes | ❌ No | User provides in UI | +| `user-or-llm` | ✅ Yes | ✅ Yes | User provides OR LLM generates | +| `llm-only` | ❌ No | ✅ Yes | LLM generates only | +| `hidden` | ❌ No | ❌ No | Application injects at runtime | #### Visibility Guidelines