fix(tables): route large CSV imports to the background job instead of 413 (#4927)

TheodoreSpeaks · web-flow · commit 2f70188091fd · 2026-06-09T14:35:41.000-04:00
* fix(tables): route large CSV imports to the background job instead of 413

* fix(tables): drop duplicate error toast on async import failure

* fix(tables): guard importId on async cancel and drop mutation objects from deps
diff --git a/apps/sim/app/workspace/[workspaceId]/tables/components/import-csv-dialog/import-csv-dialog.tsx b/apps/sim/app/workspace/[workspaceId]/tables/components/import-csv-dialog/import-csv-dialog.tsx
@@ -23,14 +23,26 @@ import {
   TableRow,
   toast,
 } from '@/components/emcn'
+import { CSV_ASYNC_IMPORT_THRESHOLD_BYTES } from '@/lib/table/constants'
 import { buildAutoMapping, parseCsvBuffer } from '@/lib/table/import'
 import type { TableDefinition } from '@/lib/table/types'
-import { type CsvImportMode, useImportCsvIntoTable } from '@/hooks/queries/tables'
+import {
+  type CsvImportMode,
+  cancelTableImport,
+  useImportCsvIntoTable,
+  useImportCsvIntoTableAsync,
+} from '@/hooks/queries/tables'
+import { useImportTrayStore } from '@/stores/table/import-tray/store'
 
 const logger = createLogger('ImportCsvDialog')
 
 const MAX_SAMPLE_ROWS = 5
 const MAX_EXAMPLES_IN_ERROR = 3
+/**
+ * Bytes read for the preview/mapping. We never parse the whole file client-side — the importer
+ * streams it server-side and the DB row-count trigger enforces the row limit.
+ */
+const CSV_PREVIEW_BYTES = 512 * 1024
 /**
  * Sentinel value for the "Do not import" option in the mapping combobox. The
  * whitespace is intentional: valid column names must match `NAME_PATTERN`
@@ -92,7 +104,18 @@ interface ParsedCsv {
   file: File
   headers: string[]
   sampleRows: Record<string, unknown>[]
-  totalRows: number
+}
+
+/** Parses the head of a CSV/TSV for the mapping + sample, dropping any truncated final line. */
+async function parseCsvPreview(file: File, delimiter: ',' | '\t') {
+  const sliced = file.size > CSV_PREVIEW_BYTES
+  const blob = sliced ? file.slice(0, CSV_PREVIEW_BYTES) : file
+  let bytes = new Uint8Array(await blob.arrayBuffer())
+  if (sliced) {
+    const lastNewline = bytes.lastIndexOf(0x0a)
+    if (lastNewline > 0) bytes = bytes.subarray(0, lastNewline + 1)
+  }
+  return parseCsvBuffer(bytes, delimiter)
 }
 
 export function ImportCsvDialog({
@@ -110,6 +133,7 @@ export function ImportCsvDialog({
   const [createHeaders, setCreateHeaders] = useState<Set<string>>(new Set())
   const [mode, setMode] = useState<CsvImportMode>('append')
   const importMutation = useImportCsvIntoTable()
+  const importAsyncMutation = useImportCsvIntoTableAsync()
 
   function resetState() {
     setParsed(null)
@@ -155,15 +179,13 @@ export function ImportCsvDialog({
     setParsing(true)
     setParseError(null)
     try {
-      const arrayBuffer = await file.arrayBuffer()
-      const delimiter = ext === 'tsv' ? '\t' : ','
-      const { headers, rows } = await parseCsvBuffer(new Uint8Array(arrayBuffer), delimiter)
+      const delimiter: ',' | '\t' = ext === 'tsv' ? '\t' : ','
+      const { headers, rows } = await parseCsvPreview(file, delimiter)
       const autoMapping = buildAutoMapping(headers, table.schema)
       setParsed({
         file,
         headers,
         sampleRows: rows.slice(0, MAX_SAMPLE_ROWS),
-        totalRows: rows.length,
       })
       setMapping(autoMapping)
     } catch (err) {
@@ -256,36 +278,70 @@ export function ImportCsvDialog({
     }
   }, [mapping, parsed?.headers, table.schema.columns, createHeaders])
 
-  const appendCapacityDeficit =
-    parsed && mode === 'append' && table.rowCount + parsed.totalRows > table.maxRows
-      ? table.rowCount + parsed.totalRows - table.maxRows
-      : 0
-
-  const replaceCapacityDeficit =
-    parsed && mode === 'replace' && parsed.totalRows > table.maxRows
-      ? parsed.totalRows - table.maxRows
-      : 0
-
   const canSubmit =
     parsed !== null &&
     !importMutation.isPending &&
+    !importAsyncMutation.isPending &&
     missingRequired.length === 0 &&
     duplicateTargets.length === 0 &&
-    mappedCount + createCount > 0 &&
-    appendCapacityDeficit === 0 &&
-    replaceCapacityDeficit === 0
+    mappedCount + createCount > 0
 
   async function handleSubmit() {
     if (!parsed || !canSubmit) return
     setSubmitError(null)
+    const createColumns = createHeaders.size > 0 ? [...createHeaders] : undefined
+
+    // Large files can't be POSTed through the server (request-body cap) — upload them
+    // straight to storage and import in the background instead. Seed the header tray and
+    // close the dialog immediately so the indicator is visible during the upload, then run
+    // the upload + kickoff in the background (don't block the dialog on it).
+    if (parsed.file.size >= CSV_ASYNC_IMPORT_THRESHOLD_BYTES) {
+      useImportTrayStore.getState().startUpload({
+        uploadId: table.id,
+        workspaceId,
+        title: parsed.file.name,
+      })
+      onOpenChange(false)
+      toast.success(`Importing "${parsed.file.name}" into "${table.name}" in the background`)
+      importAsyncMutation.mutate(
+        {
+          workspaceId,
+          tableId: table.id,
+          file: parsed.file,
+          mode,
+          mapping,
+          createColumns,
+          onProgress: (percent) => {
+            useImportTrayStore.getState().setUploadPercent(table.id, percent)
+          },
+        },
+        {
+          onSuccess: (data) => {
+            useImportTrayStore.getState().endUpload(table.id)
+            // The server row drives the tray once the list refetches. If canceled mid-upload, flag
+            // the id so it's not shown and cancel the worker server-side.
+            if (useImportTrayStore.getState().consumeCanceled(table.id) && data?.importId) {
+              useImportTrayStore.getState().cancel(table.id)
+              void cancelTableImport(workspaceId, table.id, data.importId).catch(() => {})
+            }
+          },
+          onError: () => {
+            // The hook's onError surfaces the toast; just clear the tray indicator here.
+            useImportTrayStore.getState().endUpload(table.id)
+          },
+        }
+      )
+      return
+    }
+
     try {
       const result = await importMutation.mutateAsync({
         workspaceId,
         tableId: table.id,
         file: parsed.file,
         mode,
         mapping,
-        createColumns: createHeaders.size > 0 ? [...createHeaders] : undefined,
+        createColumns,
       })
       const data = result.data
       if (mode === 'append') {
@@ -307,11 +363,7 @@ export function ImportCsvDialog({
     }
   }
 
-  const hasWarning =
-    missingRequired.length > 0 ||
-    duplicateTargets.length > 0 ||
-    appendCapacityDeficit > 0 ||
-    replaceCapacityDeficit > 0
+  const hasWarning = missingRequired.length > 0 || duplicateTargets.length > 0
 
   return (
     <ChipModal
@@ -344,7 +396,7 @@ export function ImportCsvDialog({
                   {parsed.file.name}
                 </span>
                 <span className='text-[var(--text-tertiary)] text-xs'>
-                  {parsed.totalRows.toLocaleString()} rows · {parsed.headers.length} columns
+                  {parsed.headers.length} columns
                 </span>
               </div>
               <Button variant='ghost' size='sm' onClick={resetState}>
@@ -442,20 +494,6 @@ export function ImportCsvDialog({
                     Multiple CSV columns target: {duplicateTargets.join(', ')} (pick one)
                   </p>
                 )}
-                {appendCapacityDeficit > 0 && (
-                  <p className='text-[var(--text-error)] text-caption leading-tight'>
-                    Append would exceed the row limit ({table.maxRows.toLocaleString()}) by{' '}
-                    {appendCapacityDeficit.toLocaleString()} row(s). Remove rows or switch to
-                    Replace.
-                  </p>
-                )}
-                {replaceCapacityDeficit > 0 && (
-                  <p className='text-[var(--text-error)] text-caption leading-tight'>
-                    CSV has {parsed.totalRows.toLocaleString()} rows, which exceeds the table limit
-                    of {table.maxRows.toLocaleString()} by {replaceCapacityDeficit.toLocaleString()}
-                    .
-                  </p>
-                )}
               </div>
             )}
 
diff --git a/apps/sim/app/workspace/[workspaceId]/tables/tables.tsx b/apps/sim/app/workspace/[workspaceId]/tables/tables.tsx
@@ -2,12 +2,13 @@
 
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 import { createLogger } from '@sim/logger'
+import { generateId } from '@sim/utils/id'
 import { useParams, useRouter } from 'next/navigation'
 import type { ComboboxOption } from '@/components/emcn'
 import { ChipCombobox, ChipConfirmModal, toast, Upload } from '@/components/emcn'
 import { Columns3, Rows3, Table as TableIcon } from '@/components/emcn/icons'
 import type { TableDefinition } from '@/lib/table'
-import { generateUniqueTableName } from '@/lib/table/constants'
+import { CSV_ASYNC_IMPORT_THRESHOLD_BYTES, generateUniqueTableName } from '@/lib/table/constants'
 import type {
   FilterTag,
   ResourceColumn,
@@ -24,14 +25,17 @@ import {
 import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider'
 import {
   ImportCsvDialog,
+  ImportProgressMenu,
   TablesListContextMenu,
 } from '@/app/workspace/[workspaceId]/tables/components'
 import { TableContextMenu } from '@/app/workspace/[workspaceId]/tables/components/table-context-menu'
 import { useContextMenu } from '@/app/workspace/[workspaceId]/w/components/sidebar/hooks'
 import {
+  cancelTableImport,
   downloadTableExport,
   useCreateTable,
   useDeleteTable,
+  useImportCsvAsync,
   useRenameTable,
   useTablesList,
   useUploadCsvToTable,
@@ -40,6 +44,7 @@ import { useWorkspaceMembersQuery } from '@/hooks/queries/workspace'
 import { useDebounce } from '@/hooks/use-debounce'
 import { useInlineRename } from '@/hooks/use-inline-rename'
 import { usePermissionConfig } from '@/hooks/use-permission-config'
+import { useImportTrayStore } from '@/stores/table/import-tray/store'
 
 const logger = createLogger('Tables')
 
@@ -76,6 +81,7 @@ export function Tables() {
   const renameTable = useRenameTable(workspaceId)
   const createTable = useCreateTable(workspaceId)
   const uploadCsv = useUploadCsvToTable()
+  const importCsvAsync = useImportCsvAsync()
 
   const tableRename = useInlineRename({
     onSave: (tableId, name) => renameTable.mutate({ tableId, name }),
@@ -407,37 +413,80 @@ export function Tables() {
       const list = e.target.files
       if (!list || list.length === 0 || !workspaceId) return
 
-      try {
-        setUploading(true)
+      const csvFiles = Array.from(list).filter((f) => {
+        const ext = f.name.split('.').pop()?.toLowerCase()
+        return ext === 'csv' || ext === 'tsv'
+      })
+
+      if (csvFiles.length === 0) {
+        toast.error('No CSV or TSV files selected')
+        if (csvInputRef.current) csvInputRef.current.value = ''
+        return
+      }
 
-        const csvFiles = Array.from(list).filter((f) => {
-          const ext = f.name.split('.').pop()?.toLowerCase()
-          return ext === 'csv' || ext === 'tsv'
-        })
+      // Large files can't be POSTed through the server (request-body cap) — upload them
+      // straight to storage and import in the background. These are tracked by the import
+      // tray, never the header upload button, so don't touch uploading/uploadProgress here.
+      const asyncFiles = csvFiles.filter((f) => f.size >= CSV_ASYNC_IMPORT_THRESHOLD_BYTES)
+      const syncFiles = csvFiles.filter((f) => f.size < CSV_ASYNC_IMPORT_THRESHOLD_BYTES)
 
-        if (csvFiles.length === 0) {
-          toast.error('No CSV or TSV files selected')
-          return
+      try {
+        for (const file of asyncFiles) {
+          // Show the indicator immediately under a temporary id (the real table id doesn't
+          // exist until kickoff returns), then let the tray track it. Don't redirect — the
+          // table is still empty/importing, so stay on the list.
+          const pendingId = `pending_${generateId()}`
+          useImportTrayStore
+            .getState()
+            .startUpload({ uploadId: pendingId, workspaceId, title: file.name })
+          toast.success(`Importing "${file.name}" in the background`)
+          try {
+            const result = await importCsvAsync.mutateAsync({
+              workspaceId,
+              file,
+              onProgress: (percent) => {
+                useImportTrayStore.getState().setUploadPercent(pendingId, percent)
+              },
+            })
+            useImportTrayStore.getState().endUpload(pendingId)
+            // The server row drives the tray once the list refetches (mutation invalidates it).
+            // If canceled mid-upload, flag the real id so it's not shown and cancel server-side.
+            if (
+              result?.tableId &&
+              result.importId &&
+              useImportTrayStore.getState().consumeCanceled(pendingId)
+            ) {
+              useImportTrayStore.getState().cancel(result.tableId)
+              void cancelTableImport(workspaceId, result.tableId, result.importId).catch(() => {})
+            }
+          } catch {
+            // The hook's onError surfaces the toast; just clear the tray indicator here.
+            useImportTrayStore.getState().endUpload(pendingId)
+          }
         }
 
-        setUploadProgress({ completed: 0, total: csvFiles.length })
+        if (syncFiles.length === 0) return
+
+        setUploading(true)
+        setUploadProgress({ completed: 0, total: syncFiles.length })
         const failed: string[] = []
 
-        for (let i = 0; i < csvFiles.length; i++) {
+        for (let i = 0; i < syncFiles.length; i++) {
+          const file = syncFiles[i]
           try {
-            const result = await uploadCsv.mutateAsync({ workspaceId, file: csvFiles[i] })
+            const result = await uploadCsv.mutateAsync({ workspaceId, file })
 
-            if (csvFiles.length === 1) {
+            if (syncFiles.length === 1 && asyncFiles.length === 0) {
               const tableId = result?.data?.table?.id
               if (tableId) {
                 router.push(`/workspace/${workspaceId}/tables/${tableId}`)
               }
             }
           } catch (err) {
-            failed.push(csvFiles[i].name)
+            failed.push(file.name)
             logger.error('Error uploading CSV:', err)
           } finally {
-            setUploadProgress({ completed: i + 1, total: csvFiles.length })
+            setUploadProgress({ completed: i + 1, total: syncFiles.length })
           }
         }
 
@@ -459,7 +508,8 @@ export function Tables() {
         }
       }
     },
-    [workspaceId, router, uploadCsv]
+    // eslint-disable-next-line react-hooks/exhaustive-deps -- mutation objects are unstable; mutateAsync is stable in v5
+    [workspaceId, router]
   )
 
   const handleListUploadCsv = useCallback(() => {
@@ -508,6 +558,7 @@ export function Tables() {
         sort={sortConfig}
         filter={filterContent}
         filterTags={filterTags}
+        leadingActions={<ImportProgressMenu workspaceId={workspaceId} />}
         headerActions={[
           {
             label: uploadButtonLabel,
