Merge remote-tracking branch 'origin/staging' into improvement/platform

Author: waleedlatif1

apps/docs/content/docs/en/triggers/table.mdx

@@ -38,7 +38,6 @@ Triggers when rows are inserted or updated in a table
 | `changedColumns` | json | List of column names that changed \(empty for inserts\) |
 | `rowId` | string | The unique row ID |
 | `headers` | json | Column names from the table schema |
-| `rowNumber` | number | The position of the row in the table |
 | `tableId` | string | The table ID |
 | `tableName` | string | The table name |
 | `timestamp` | string | Event timestamp in ISO format |

apps/sim/app/api/table/[tableId]/rows/route.ts

@@ -71,6 +71,7 @@ async function handleBatchInsert(
         workspaceId: validated.workspaceId,
         userId,
         positions: validated.positions,
+        orderKeys: validated.orderKeys,
       },
       table,
       requestId
@@ -83,6 +84,7 @@ async function handleBatchInsert(
           id: r.id,
           data: r.data,
           position: r.position,
+          orderKey: r.orderKey ?? undefined,
           createdAt: r.createdAt instanceof Date ? r.createdAt.toISOString() : r.createdAt,
           updatedAt: r.updatedAt instanceof Date ? r.updatedAt.toISOString() : r.updatedAt,
         })),
@@ -162,6 +164,8 @@ export const POST = withRouteHandler(
           workspaceId: validated.workspaceId,
           userId: authResult.userId,
           position: validated.position,
+          afterRowId: validated.afterRowId,
+          beforeRowId: validated.beforeRowId,
         },
         table,
         requestId
@@ -174,9 +178,11 @@ export const POST = withRouteHandler(
             id: row.id,
             data: row.data,
             position: row.position,
+            orderKey: row.orderKey ?? undefined,
             createdAt: row.createdAt instanceof Date ? row.createdAt.toISOString() : row.createdAt,
             updatedAt: row.updatedAt instanceof Date ? row.updatedAt.toISOString() : row.updatedAt,
           },
+
           message: 'Row inserted successfully',
         },
       })

apps/sim/app/api/tools/clickhouse/utils.ts

@@ -1,4 +1,7 @@
-import { validateDatabaseHost } from '@/lib/core/security/input-validation.server'
+import {
+  validateDatabaseHost,
+  validateSqlWhereClause,
+} from '@/lib/core/security/input-validation.server'
 import type { ClickHouseConnectionConfig } from '@/tools/clickhouse/types'
 
 const REQUEST_TIMEOUT_MS = 30_000
@@ -60,7 +63,8 @@ export interface ClickHouseIntrospectionResult {
  */
 async function clickhouseRequest(
   config: ClickHouseConnectionConfig,
-  statement: string
+  statement: string,
+  options: { readOnly?: boolean } = {}
 ): Promise<ClickHouseHttpResult> {
   const hostValidation = await validateDatabaseHost(config.host, 'host')
   if (!hostValidation.isValid) {
@@ -70,6 +74,12 @@ async function clickhouseRequest(
   const protocol = config.secure ? 'https' : 'http'
   const url = new URL(`${protocol}://${config.host}:${config.port}/`)
   url.searchParams.set('database', config.database)
+  if (options.readOnly) {
+    // Server-enforced read-only: ClickHouse rejects any write/DDL and forbids the
+    // query from re-enabling writes via `SET readonly=0`. This is the real boundary
+    // for the query operation; the SQL-shape checks below are defense-in-depth.
+    url.searchParams.set('readonly', '1')
+  }
 
   const controller = new AbortController()
   const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS)
@@ -290,7 +300,9 @@ export async function executeClickHouseQuery(
       )
     }
   }
-  const result = await clickhouseRequest(config, ensureJsonFormat(query))
+  const result = await clickhouseRequest(config, ensureJsonFormat(query), {
+    readOnly: options.enforceReadOnly,
+  })
   return parseRowsResult(result)
 }
 
@@ -448,37 +460,14 @@ function sanitizeSingleIdentifier(identifier: string): string {
 }
 
 /**
- * Rejects WHERE clauses containing patterns commonly used in SQL injection so
- * that user-supplied conditions cannot escape the intended mutation.
+ * Rejects WHERE clauses containing SQL-injection or always-true tautology
+ * patterns so user-supplied conditions cannot broaden a mutation to every row.
+ * Delegates to the shared {@link validateSqlWhereClause} guard (defense-in-depth).
  */
 function validateWhereClause(where: string): void {
-  const dangerousPatterns = [
-    /;\s*(drop|delete|insert|alter|create|truncate|rename|grant|revoke)/i,
-    /union\s+(all\s+)?select/i,
-    /into\s+outfile/i,
-    /--/,
-    /\/\*/,
-    /\*\//,
-    /\bor\s+(['"]?)(\w+)\1\s*=\s*\1\2\1/i,
-    /\bor\s+true\b/i,
-    /\bor\s+false\b/i,
-    /\band\s+(['"]?)(\w+)\1\s*=\s*\1\2\1/i,
-    /\band\s+true\b/i,
-    /\band\s+false\b/i,
-    /\bsleep\s*\(/i,
-    /;\s*\w+/,
-    // Constant / tautological conditions that don't reference columns and would
-    // broaden a mutation to all rows (e.g. "1=1", "1 < 2", "'a'='a'", bare "1"/"true").
-    /\b\d+\s*(?:=|==|<>|!=|<=|>=|<|>)\s*\d+\b/,
-    /(['"])([^'"]*)\1\s*(?:=|==|<>|!=)\s*\1\2\1/,
-    /\b(\w+)\s*=\s*\1\b/i,
-    /^\s*(?:\d+|true|false)\s*$/i,
-  ]
-
-  for (const pattern of dangerousPatterns) {
-    if (pattern.test(where)) {
-      throw new Error('WHERE clause contains potentially dangerous operation')
-    }
+  const result = validateSqlWhereClause(where, 'WHERE clause')
+  if (!result.isValid) {
+    throw new Error(result.error)
   }
 }
 

apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/table-grid.tsx

@@ -837,9 +837,12 @@ export function TableGrid({
 
   function handleInsertRow(offset: 0 | 1) {
     if (!contextMenu.row) return
+    const anchorId = contextMenu.row.id
+    // Fractional ordering: express intent by neighbor id, not integer position.
+    const intent = offset === 0 ? { beforeRowId: anchorId } : { afterRowId: anchorId }
     const position = contextMenu.row.position + offset
     createRef.current(
-      { data: {}, position },
+      { data: {}, ...intent },
       {
         onSuccess: (response: Record<string, unknown>) => {
           const newRowId = extractCreatedRowId(response)
@@ -904,7 +907,7 @@ export function TableGrid({
     const sourceArrayIndex = rowsRef.current.findIndex((r) => r.id === contextRow.id)
     closeContextMenu()
     createRef.current(
-      { data: rowData, position },
+      { data: rowData, afterRowId: contextRow.id },
       {
         onSuccess: (response: Record<string, unknown>) => {
           const newRowId = extractCreatedRowId(response)

apps/sim/app/workspace/[workspaceId]/tables/[tableId]/components/table-grid/utils.ts

@@ -302,7 +302,12 @@ export function computeNormalizedSelection(
 export function collectRowSnapshots(rows: Iterable<TableRowType>): DeletedRowSnapshot[] {
   const snapshots: DeletedRowSnapshot[] = []
   for (const row of rows) {
-    snapshots.push({ rowId: row.id, data: { ...row.data }, position: row.position })
+    snapshots.push({
+      rowId: row.id,
+      data: { ...row.data },
+      position: row.position,
+      orderKey: row.orderKey,
+    })
   }
   return snapshots
 }

apps/sim/background/workflow-column-execution.ts

@@ -589,7 +589,6 @@ async function runWorkflowAndWriteTerminal(
         changedColumns: [],
         rowId,
         headers,
-        rowNumber: row.position,
         tableId,
         tableName,
         timestamp: new Date().toISOString(),

apps/sim/hooks/queries/tables.ts

@@ -512,7 +512,13 @@ export function useCreateTableRow({ workspaceId, tableId }: RowMutationContext)
     ) => {
       return requestJson(createTableRowContract, {
         params: { tableId },
-        body: { workspaceId, data: variables.data as RowData, position: variables.position },
+        body: {
+          workspaceId,
+          data: variables.data as RowData,
+          position: variables.position,
+          afterRowId: variables.afterRowId,
+          beforeRowId: variables.beforeRowId,
+        },
       })
     },
     onSuccess: (response) => {
@@ -592,35 +598,47 @@ function reconcileCreatedRow(
       if (!old) return old
       if (old.pages.some((p) => p.rows.some((r) => r.id === row.id))) return old
 
-      const pages = old.pages.map((page) =>
-        page.rows.some((r) => r.position >= row.position)
-          ? {
-              ...page,
-              rows: page.rows.map((r) =>
-                r.position >= row.position ? { ...r, position: r.position + 1 } : r
-              ),
-            }
-          : page
-      )
+      // Use key-ordering only when the new row AND every cached row have an
+      // `orderKey` — then no neighbor bump is needed and order is exact. If any
+      // cached row is un-keyed (mid-backfill), fall back to the legacy `position`
+      // path so un-keyed rows aren't yanked to the front by an empty-string sort.
+      const byKey =
+        row.orderKey != null && old.pages.every((p) => p.rows.every((r) => r.orderKey != null))
+      const sortRows = (rows: TableRow[]) =>
+        byKey
+          ? [...rows].sort((a, b) => (a.orderKey as string).localeCompare(b.orderKey as string))
+          : [...rows].sort((a, b) => a.position - b.position)
+      const fitsAfter = (last: TableRow | undefined) =>
+        last === undefined ||
+        (byKey
+          ? (last.orderKey as string) >= (row.orderKey as string)
+          : last.position >= row.position)
+
+      const pages = byKey
+        ? old.pages
+        : old.pages.map((page) =>
+            page.rows.some((r) => r.position >= row.position)
+              ? {
+                  ...page,
+                  rows: page.rows.map((r) =>
+                    r.position >= row.position ? { ...r, position: r.position + 1 } : r
+                  ),
+                }
+              : page
+          )
 
       let inserted = false
       const nextPages = pages.map((page) => {
         if (inserted) return page
-        const last = page.rows[page.rows.length - 1]
-        const fits = last === undefined || last.position >= row.position
-        if (!fits) return page
+        if (!fitsAfter(page.rows[page.rows.length - 1])) return page
         inserted = true
-        const merged = [...page.rows, row].sort((a, b) => a.position - b.position)
-        return { ...page, rows: merged }
+        return { ...page, rows: sortRows([...page.rows, row]) }
       })
 
       if (!inserted && nextPages.length > 0) {
         const lastIdx = nextPages.length - 1
         const lastPage = nextPages[lastIdx]
-        nextPages[lastIdx] = {
-          ...lastPage,
-          rows: [...lastPage.rows, row].sort((a, b) => a.position - b.position),
-        }
+        nextPages[lastIdx] = { ...lastPage, rows: sortRows([...lastPage.rows, row]) }
       }
 
       const firstPage = nextPages[0]
@@ -655,6 +673,7 @@ export function useBatchCreateTableRows({ workspaceId, tableId }: RowMutationCon
           workspaceId,
           rows: variables.rows as RowData[],
           positions: variables.positions,
+          orderKeys: variables.orderKeys,
         },
       })
     },

apps/sim/hooks/use-table-undo.ts

@@ -5,7 +5,6 @@ import {
   useAddTableColumn,
   useBatchCreateTableRows,
   useBatchUpdateTableRows,
-  useCreateTableRow,
   useDeleteColumn,
   useDeleteTableRow,
   useDeleteTableRows,
@@ -56,7 +55,6 @@ export function useTableUndo({
   const canRedo = useTableUndoStore((s) => (s.stacks[tableId]?.redo.length ?? 0) > 0)
 
   const updateRowMutation = useUpdateTableRow({ workspaceId, tableId })
-  const createRowMutation = useCreateTableRow({ workspaceId, tableId })
   const batchCreateRowsMutation = useBatchCreateTableRows({ workspaceId, tableId })
   const batchUpdateRowsMutation = useBatchUpdateTableRows({ workspaceId, tableId })
   const deleteRowMutation = useDeleteTableRow({ workspaceId, tableId })
@@ -137,11 +135,18 @@ export function useTableUndo({
             if (direction === 'undo') {
               deleteRowMutation.mutate(action.rowId)
             } else {
-              createRowMutation.mutate(
-                { data: action.data ?? {}, position: action.position },
+              // Redo via the batch path so the saved orderKey restores exact placement.
+              // The single-insert API has no orderKey field, and under the fractional-ordering
+              // flag its `position` is read as a rank — a gappy saved position misplaces.
+              batchCreateRowsMutation.mutate(
+                {
+                  rows: [action.data ?? {}],
+                  positions: [action.position],
+                  orderKeys: action.orderKey ? [action.orderKey] : undefined,
+                },
                 {
                   onSuccess: (response) => {
-                    const newRowId = extractCreatedRowId(response as Record<string, unknown>)
+                    const newRowId = response?.data?.rows?.[0]?.id
                     if (newRowId && newRowId !== action.rowId) {
                       patchUndoRowId(tableId, action.rowId, newRowId)
                     }
@@ -165,6 +170,9 @@ export function useTableUndo({
                 {
                   rows: action.rows.map((r) => r.data),
                   positions: action.rows.map((r) => r.position),
+                  orderKeys: action.rows.every((r) => r.orderKey)
+                    ? action.rows.map((r) => r.orderKey as string)
+                    : undefined,
                 },
                 {
                   onSuccess: (response) => {
@@ -187,6 +195,9 @@ export function useTableUndo({
                 {
                   rows: action.rows.map((row) => row.data),
                   positions: action.rows.map((row) => row.position),
+                  orderKeys: action.rows.every((row) => row.orderKey)
+                    ? action.rows.map((row) => row.orderKey as string)
+                    : undefined,
                 },
                 {
                   onSuccess: (response) => {

apps/sim/lib/api/contracts/tables.ts

@@ -147,12 +147,29 @@ export const rowDataSchema = domainObjectSchema<RowData>()
 export const tableDefinitionSchema = domainObjectSchema<TableDefinition>()
 export const tableRowSchema = domainObjectSchema<TableRow>()
 
-export const insertTableRowBodySchema = z.object({
+/**
+ * Plain-object base for the single-row insert body. Kept un-refined so callers
+ * (e.g. the v1 public contract) can `.omit()` fields before applying
+ * {@link rowAnchorMutexRefine} — Zod forbids `.omit()` on a refined schema.
+ */
+export const insertTableRowBodyBaseSchema = z.object({
   workspaceId: z.string().min(1, 'Workspace ID is required'),
   data: rowDataSchema,
   position: z.number().int().min(0).optional(),
+  /** Fractional ordering: insert directly after this row id. Takes precedence over `position`. */
+  afterRowId: z.string().min(1).optional(),
+  /** Fractional ordering: insert directly before this row id. Takes precedence over `position`. */
+  beforeRowId: z.string().min(1).optional(),
 })
 
+/** `afterRowId` and `beforeRowId` are mutually exclusive insert anchors. */
+export const rowAnchorMutexRefine = [
+  (data: { afterRowId?: string; beforeRowId?: string }) => !data.afterRowId || !data.beforeRowId,
+  { message: 'afterRowId and beforeRowId are mutually exclusive' },
+] as const
+
+export const insertTableRowBodySchema = insertTableRowBodyBaseSchema.refine(...rowAnchorMutexRefine)
+
 /**
  * POST `/api/table/[tableId]/rows/upsert` body — insert-or-update keyed by a
  * unique column name. `conflictTarget` is optional (server picks a single
@@ -175,13 +192,18 @@ export const batchInsertTableRowsBodySchema = z
         `Cannot insert more than ${TABLE_LIMITS.MAX_BATCH_INSERT_SIZE} rows per batch`
       ),
     positions: z.array(z.number().int().min(0)).max(TABLE_LIMITS.MAX_BATCH_INSERT_SIZE).optional(),
+    /** Fractional ordering: exact per-row order keys (undo restore). Takes precedence over `positions`. */
+    orderKeys: z.array(z.string().min(1)).max(TABLE_LIMITS.MAX_BATCH_INSERT_SIZE).optional(),
   })
   .refine((data) => !data.positions || data.positions.length === data.rows.length, {
     message: 'positions array length must match rows array length',
   })
   .refine((data) => !data.positions || new Set(data.positions).size === data.positions.length, {
     message: 'positions must not contain duplicates',
   })
+  .refine((data) => !data.orderKeys || data.orderKeys.length === data.rows.length, {
+    message: 'orderKeys array length must match rows array length',
+  })
 
 /**
  * POST `/api/table/[tableId]/rows` body — accepts either a batch payload

apps/sim/lib/api/contracts/v1/tables/index.ts

@@ -4,7 +4,8 @@ import {
   createTableColumnBodySchema,
   deleteTableColumnBodySchema,
   deleteTableRowsBodySchema,
-  insertTableRowBodySchema,
+  insertTableRowBodyBaseSchema,
+  rowAnchorMutexRefine,
   rowDataSchema,
   tableIdParamsSchema,
   tableRowParamsSchema,
@@ -60,7 +61,9 @@ export const v1CreateTableBodySchema = createTableBodySchema.omit({
  * Public API insert row body — no caller-controlled `position`. Server places
  * new rows at the tail; ordering by index is an in-app affordance only.
  */
-export const v1InsertTableRowBodySchema = insertTableRowBodySchema.omit({ position: true })
+export const v1InsertTableRowBodySchema = insertTableRowBodyBaseSchema
+  .omit({ position: true })
+  .refine(...rowAnchorMutexRefine)
 
 /**
  * Public API batch insert body — no `positions`. Same rationale as above.