diff --git a/apps/web/app/(app)/settings/page.tsx b/apps/web/app/(app)/settings/page.tsx index 51745b80a..8d221f8a8 100644 --- a/apps/web/app/(app)/settings/page.tsx +++ b/apps/web/app/(app)/settings/page.tsx @@ -8,6 +8,7 @@ import { cn } from "@lib/utils" import { dmSansClassName, dmSans125ClassName } from "@/lib/fonts" import Account from "@/components/settings/account" import Billing from "@/components/settings/billing" +import { DataPortabilityPanel } from "@/components/settings/data-portability" import Integrations from "@/components/settings/integrations" import ConnectionsMCP from "@/components/settings/connections-mcp" import Support from "@/components/settings/support" @@ -24,6 +25,7 @@ import { LoaderIcon, User as UserIcon, Zap, + Download, HelpCircle, CreditCard, ShieldAlert, @@ -49,6 +51,7 @@ const TABS = [ "account", "billing", "integrations", + "portability", "connections", "support", ] as const @@ -80,6 +83,12 @@ const NAV_ITEMS: NavItem[] = [ description: "Save, sync and search across tools", icon: , }, + { + id: "portability", + label: "Data portability", + description: "Export and restore your memories", + icon: , + }, { id: "connections", label: "Connections & MCP", @@ -572,6 +581,7 @@ export default function SettingsPage() { {activeTab === "account" && } {activeTab === "billing" && } {activeTab === "integrations" && } + {activeTab === "portability" && } {activeTab === "connections" && } {activeTab === "support" && } diff --git a/apps/web/app/api/memories/_utils.ts b/apps/web/app/api/memories/_utils.ts new file mode 100644 index 000000000..bc0764a24 --- /dev/null +++ b/apps/web/app/api/memories/_utils.ts @@ -0,0 +1,62 @@ +const BACKEND_BASE_URL = + process.env.NEXT_PUBLIC_BACKEND_URL ?? "https://api.supermemory.ai" + +export function buildBackendHeaders(request: Request): Headers { + const headers = new Headers({ + "Content-Type": "application/json", + "X-App-Source": "nova", + }) + + const cookie = request.headers.get("cookie") + if (cookie) { + headers.set("cookie", cookie) + } + + const authorization = request.headers.get("authorization") + if (authorization) { + headers.set("authorization", authorization) + } + + return headers +} + +export function getBackendBaseUrl() { + return BACKEND_BASE_URL +} + +export function chunkArray(items: T[], size: number): T[][] { + if (size <= 0) return [items] + const chunks: T[][] = [] + for (let index = 0; index < items.length; index += size) { + chunks.push(items.slice(index, index + size)) + } + return chunks +} + +export function parseContainerTags(searchParams: URLSearchParams): string[] { + const values = searchParams.getAll("containerTags") + if (values.length === 0) return [] + + return [...new Set(values.flatMap((value) => value.split(",")))] + .map((value) => value.trim()) + .filter(Boolean) +} + +export function parseDateBound(value: string, bound: "start" | "end") { + const date = new Date(value) + if (Number.isNaN(date.getTime())) return null + + if (!value.includes("T") && bound === "end") { + date.setUTCHours(23, 59, 59, 999) + } + + if (!value.includes("T") && bound === "start") { + date.setUTCHours(0, 0, 0, 0) + } + + return date +} + +export function escapeMarkdown(value: string) { + return value.replace(/[\\`*_{}\[\]()#+\-.!|>]/g, "\\$&") +} diff --git a/apps/web/app/api/memories/export/route.ts b/apps/web/app/api/memories/export/route.ts new file mode 100644 index 000000000..b1f9f89da --- /dev/null +++ b/apps/web/app/api/memories/export/route.ts @@ -0,0 +1,207 @@ +import { NextResponse } from "next/server" +import { + buildBackendHeaders, + escapeMarkdown, + getBackendBaseUrl, + parseContainerTags, + parseDateBound, +} from "../_utils" + +type ExportDocument = { + id: string + customId?: string | null + content?: string | null + summary?: string | null + title?: string | null + url?: string | null + source?: string | null + type?: string | null + status?: string | null + metadata?: Record | null + containerTags?: string[] | null + createdAt?: string | Date + updatedAt?: string | Date + memoryEntries?: Array<{ + id?: string + memory?: string + isStatic?: boolean + createdAt?: string | Date + }> +} + +type DocumentsResponse = { + documents?: ExportDocument[] + pagination?: { + currentPage?: number + totalPages?: number + totalItems?: number + limit?: number + } +} + +async function fetchDocumentsPage( + request: Request, + page: number, + limit: number, + containerTags: string[], +) { + const response = await fetch( + `${getBackendBaseUrl()}/v3/documents/documents`, + { + method: "POST", + headers: buildBackendHeaders(request), + body: JSON.stringify({ + page, + limit, + order: "asc", + sort: "createdAt", + ...(containerTags.length > 0 ? { containerTags } : {}), + }), + }, + ) + + if (!response.ok) { + const message = await response.text() + throw new Error(message || `Export failed with status ${response.status}`) + } + + return (await response.json()) as DocumentsResponse +} + +function buildMarkdownExport(documents: ExportDocument[], exportedAt: string) { + const lines: string[] = [ + "# supermemory memory export", + "", + `- Exported at: ${exportedAt}`, + `- Memory count: ${documents.length}`, + "", + ] + + for (const document of documents) { + const heading = document.title?.trim() || document.customId || document.id + const createdAt = + document.createdAt instanceof Date + ? document.createdAt.toISOString() + : document.createdAt + ? new Date(document.createdAt).toISOString() + : exportedAt + const tags = document.containerTags?.length + ? document.containerTags.join(", ") + : "None" + const content = (document.content ?? document.summary ?? "").trim() + const memoryLines = (document.memoryEntries ?? []).map((entry) => { + const prefix = entry.isStatic ? "static" : "dynamic" + return `- ${prefix}: ${entry.memory ?? ""}`.trim() + }) + + lines.push(`## ${escapeMarkdown(heading)}`) + lines.push("") + lines.push(`- ID: ${document.id}`) + if (document.customId) lines.push(`- Custom ID: ${document.customId}`) + lines.push(`- Created: ${createdAt}`) + if (document.updatedAt) { + const updatedAt = + document.updatedAt instanceof Date + ? document.updatedAt.toISOString() + : new Date(document.updatedAt).toISOString() + lines.push(`- Updated: ${updatedAt}`) + } + lines.push(`- Tags: ${tags}`) + if (document.type) lines.push(`- Type: ${document.type}`) + if (document.status) lines.push(`- Status: ${document.status}`) + if (document.url) lines.push(`- URL: ${document.url}`) + if (document.source) lines.push(`- Source: ${document.source}`) + lines.push("") + lines.push("### Content") + lines.push("") + lines.push(content || "No content available.") + if (memoryLines.length > 0) { + lines.push("") + lines.push("### Memories") + lines.push("") + lines.push(...memoryLines) + } + lines.push("") + lines.push("---") + lines.push("") + } + + return lines.join("\n") +} + +export async function GET(request: Request) { + try { + const { searchParams } = new URL(request.url) + const format = searchParams.get("format") === "markdown" ? "markdown" : "json" + const startDateParam = searchParams.get("startDate") + const endDateParam = searchParams.get("endDate") + const containerTags = parseContainerTags(searchParams) + const pageSize = 100 + + const startDate = startDateParam ? parseDateBound(startDateParam, "start") : null + const endDate = endDateParam ? parseDateBound(endDateParam, "end") : null + + if (startDateParam && !startDate) { + return NextResponse.json({ error: "Invalid startDate" }, { status: 400 }) + } + + if (endDateParam && !endDate) { + return NextResponse.json({ error: "Invalid endDate" }, { status: 400 }) + } + + const firstPage = await fetchDocumentsPage(request, 1, pageSize, containerTags) + const documents = [...(firstPage.documents ?? [])] + const totalPages = firstPage.pagination?.totalPages ?? 1 + + for (let page = 2; page <= totalPages; page += 1) { + const response = await fetchDocumentsPage(request, page, pageSize, containerTags) + documents.push(...(response.documents ?? [])) + } + + const filteredDocuments = documents.filter((document) => { + const createdAt = document.createdAt ? new Date(document.createdAt) : null + if (startDate && (!createdAt || createdAt < startDate)) return false + if (endDate && (!createdAt || createdAt > endDate)) return false + return true + }) + + const exportedAt = new Date().toISOString() + const payload = { + version: 1, + exportedAt, + format, + filters: { + containerTags, + startDate: startDateParam, + endDate: endDateParam, + }, + count: filteredDocuments.length, + documents: filteredDocuments, + } + + if (format === "markdown") { + const markdown = buildMarkdownExport(filteredDocuments, exportedAt) + return new NextResponse(markdown, { + headers: { + "Content-Type": "text/markdown; charset=utf-8", + "Content-Disposition": `attachment; filename="supermemory-export-${exportedAt}.md"`, + }, + }) + } + + return new NextResponse(JSON.stringify(payload, null, 2), { + headers: { + "Content-Type": "application/json; charset=utf-8", + "Content-Disposition": `attachment; filename="supermemory-export-${exportedAt}.json"`, + }, + }) + } catch (error) { + console.error("Memory export failed:", error) + return NextResponse.json( + { + error: error instanceof Error ? error.message : "Failed to export memories", + }, + { status: 500 }, + ) + } +} diff --git a/apps/web/app/api/memories/import/route.ts b/apps/web/app/api/memories/import/route.ts new file mode 100644 index 000000000..03cd87a84 --- /dev/null +++ b/apps/web/app/api/memories/import/route.ts @@ -0,0 +1,114 @@ +import { NextResponse } from "next/server" +import { buildBackendHeaders, chunkArray, getBackendBaseUrl } from "../_utils" + +type ExportDocument = { + content?: string | null + summary?: string | null + customId?: string | null + containerTags?: string[] | null + metadata?: Record | null + entityContext?: string | null +} + +type ImportPayload = + | { + containerTags?: string[] + targetContainerTags?: string[] + documents?: ExportDocument[] + memories?: ExportDocument[] + data?: ExportDocument[] + } + | ExportDocument[] + +function normalizeDocuments(payload: ImportPayload): ExportDocument[] { + if (Array.isArray(payload)) return payload + if (Array.isArray(payload.documents)) return payload.documents + if (Array.isArray(payload.memories)) return payload.memories + if (Array.isArray(payload.data)) return payload.data + return [] +} + +function pickTargetContainerTags(payload: ImportPayload): string[] | null { + if (Array.isArray(payload)) return null + if (Array.isArray(payload.targetContainerTags) && payload.targetContainerTags.length > 0) { + return payload.targetContainerTags + } + if (Array.isArray(payload.containerTags) && payload.containerTags.length > 0) { + return payload.containerTags + } + return null +} + +export async function POST(request: Request) { + try { + const payload = (await request.json()) as ImportPayload + const documents = normalizeDocuments(payload) + const overrideContainerTags = pickTargetContainerTags(payload) + + if (documents.length === 0) { + return NextResponse.json( + { error: "No memories were provided for import" }, + { status: 400 }, + ) + } + + const batches = chunkArray( + documents + .map((document) => { + const content = (document.content ?? document.summary ?? "").trim() + if (!content) return null + return { + content, + customId: document.customId ?? undefined, + containerTags: + overrideContainerTags ?? document.containerTags ?? undefined, + metadata: document.metadata ?? undefined, + entityContext: document.entityContext ?? undefined, + } + }) + .filter((document): document is NonNullable => document !== null), + 25, + ) + + if (batches.length === 0 || batches.every((batch) => batch.length === 0)) { + return NextResponse.json( + { error: "No importable memories were found in the payload" }, + { status: 400 }, + ) + } + + let imported = 0 + for (const batch of batches) { + if (batch.length === 0) continue + + const response = await fetch(`${getBackendBaseUrl()}/v3/documents/batch`, { + method: "POST", + headers: buildBackendHeaders(request), + body: JSON.stringify({ + documents: batch, + metadata: { + sm_source: "supermemory-export", + sm_imported_at: new Date().toISOString(), + }, + }), + }) + + if (!response.ok) { + const message = await response.text() + throw new Error(message || `Import failed with status ${response.status}`) + } + + imported += batch.length + } + + return NextResponse.json({ success: true, imported }) + } catch (error) { + console.error("Memory import failed:", error) + return NextResponse.json( + { + error: error instanceof Error ? error.message : "Failed to import memories", + }, + { status: 500 }, + ) + } +} diff --git a/apps/web/components/settings/data-portability.tsx b/apps/web/components/settings/data-portability.tsx new file mode 100644 index 000000000..cf406d2a8 --- /dev/null +++ b/apps/web/components/settings/data-portability.tsx @@ -0,0 +1,399 @@ +"use client" + +import { dmSans125ClassName } from "@/lib/fonts" +import { cn } from "@lib/utils" +import { useAuth } from "@lib/auth-context" +import { useContainerTags } from "@/hooks/use-container-tags" +import { + Check, + Download, + FileDown, + LoaderIcon, + Upload, +} from "lucide-react" +import { useMemo, useRef, useState } from "react" +import { toast } from "sonner" + +type ExportFormat = "json" | "markdown" + +function SectionTitle({ children }: { children: React.ReactNode }) { + return ( +

+ {children} +

+ ) +} + +function SettingsCard({ children }: { children: React.ReactNode }) { + return ( +
+ {children} +
+ ) +} + +function PillButton({ + children, + onClick, + disabled, + className, +}: { + children: React.ReactNode + onClick?: () => void + disabled?: boolean + className?: string +}) { + return ( + + ) +} + +function formatFilterLabel(value: string) { + return value.replace(/_/g, " ") +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +function getDownloadFilename(response: Response, fallback: string) { + const disposition = response.headers.get("content-disposition") + if (!disposition) return fallback + + const filenameMatch = disposition.match(/filename="?([^";]+)"?/i) + return filenameMatch?.[1] ?? fallback +} + +export function DataPortabilityPanel() { + const { org } = useAuth() + const { allProjects } = useContainerTags() + const [format, setFormat] = useState("json") + const [selectedTags, setSelectedTags] = useState([]) + const [startDate, setStartDate] = useState("") + const [endDate, setEndDate] = useState("") + const [isExporting, setIsExporting] = useState(false) + const [isImporting, setIsImporting] = useState(false) + const fileInputRef = useRef(null) + + const availableTags = useMemo(() => { + return [...new Set((allProjects ?? []).map((project) => project.containerTag))] + .filter(Boolean) + .sort((a, b) => a.localeCompare(b)) + }, [allProjects]) + + const canExport = !isExporting + const activeTagSummary = + selectedTags.length > 0 + ? `${selectedTags.length} selected` + : org?.id + ? `Current org: ${org.name ?? org.id}` + : "All memories" + + const toggleTag = (tag: string) => { + setSelectedTags((current) => + current.includes(tag) + ? current.filter((currentTag) => currentTag !== tag) + : [...current, tag], + ) + } + + const downloadBlob = (blob: Blob, filename: string) => { + const url = window.URL.createObjectURL(blob) + const anchor = document.createElement("a") + anchor.href = url + anchor.download = filename + document.body.appendChild(anchor) + anchor.click() + anchor.remove() + window.URL.revokeObjectURL(url) + } + + const handleExport = async (selectedFormat: ExportFormat) => { + if (!canExport) return + setIsExporting(true) + try { + const params = new URLSearchParams() + params.set("format", selectedFormat) + for (const tag of selectedTags) { + params.append("containerTags", tag) + } + if (startDate) params.set("startDate", startDate) + if (endDate) params.set("endDate", endDate) + + const response = await fetch(`/api/memories/export?${params.toString()}`, { + credentials: "include", + }) + + if (!response.ok) { + const body = await response.json().catch(() => ({})) + throw new Error( + (isPlainObject(body) && typeof body.error === "string" + ? body.error + : null) ?? "Failed to export memories", + ) + } + + const filename = getDownloadFilename( + response, + `supermemory-export.${selectedFormat === "markdown" ? "md" : "json"}`, + ) + downloadBlob(await response.blob(), filename) + toast.success("Export downloaded") + } catch (error) { + toast.error("Failed to export memories", { + description: error instanceof Error ? error.message : undefined, + }) + } finally { + setIsExporting(false) + } + } + + const handleImportFile = async (event: React.ChangeEvent) => { + const file = event.target.files?.[0] + event.target.value = "" + if (!file) return + + setIsImporting(true) + try { + const text = await file.text() + const parsed = JSON.parse(text) as unknown + + const response = await fetch("/api/memories/import", { + method: "POST", + credentials: "include", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(parsed), + }) + + if (!response.ok) { + const body = await response.json().catch(() => ({})) + throw new Error( + (isPlainObject(body) && typeof body.error === "string" + ? body.error + : null) ?? "Failed to import memories", + ) + } + + const result = (await response.json()) as { imported?: number } + toast.success("Import completed", { + description: `${result.imported ?? 0} memories imported successfully.`, + }) + } catch (error) { + toast.error("Failed to import memories", { + description: error instanceof Error ? error.message : undefined, + }) + } finally { + setIsImporting(false) + } + } + + return ( +
+
+ Data portability + +
+
+

+ Backup, migrate, or archive your memories. +

+

+ Export memories as JSON or Markdown and import previous JSON + archives later. +

+
+ +
+
+
+
+

Export memories

+

+ Choose a format, then optionally narrow by date or tags. +

+
+ +
+ +
+ setFormat("json")} + className={format === "json" ? "bg-white text-black" : "bg-white/5 text-white"} + > + JSON + + setFormat("markdown")} + className={format === "markdown" ? "bg-white text-black" : "bg-white/5 text-white"} + > + Markdown + +
+ +
+ + +
+ +
+
+

Filter by tags

+

{activeTagSummary}

+
+
+ {availableTags.length > 0 ? ( + availableTags.map((tag) => { + const isSelected = selectedTags.includes(tag) + return ( + + ) + }) + ) : ( +

+ No container tags are available yet. +

+ )} +
+
+ +
+

+ Markdown is best for human-readable backups. JSON is best for + round-tripping. +

+ handleExport(format)} + disabled={!canExport} + className="bg-[#4BA0FA] text-[#00171A]" + > + {isExporting ? ( + + ) : ( + + )} + {isExporting ? "Exporting..." : "Download export"} + +
+
+ +
+
+
+

Import memories

+

+ Upload a previously exported JSON archive to restore memories. +

+
+ +
+ +
+ +
+ +

+ Import keeps any container tags stored in the archive. If you want to + move memories into a different workspace, export the JSON and adjust + tags before importing. +

+ { + fileInputRef.current?.click() + }} + className="mt-5 bg-white/5 text-white" + disabled={isImporting} + > + {isImporting ? ( + + ) : ( + + )} + {isImporting ? "Importing..." : "Import JSON archive"} + +
+
+
+ +
+
+ ) +} diff --git a/packages/pipecat-sdk-python/src/supermemory_pipecat/service.py b/packages/pipecat-sdk-python/src/supermemory_pipecat/service.py index 2aef866bf..eb9d5fb66 100644 --- a/packages/pipecat-sdk-python/src/supermemory_pipecat/service.py +++ b/packages/pipecat-sdk-python/src/supermemory_pipecat/service.py @@ -146,14 +146,17 @@ async def _retrieve_memories(self, query: str) -> Dict[str, Any]: response = await self._supermemory_client.profile(**kwargs) + profile = getattr(response, "profile", None) + search_results_response = getattr(response, "search_results", None) + search_results = [] - if response.search_results and response.search_results.results: - search_results = response.search_results.results + if search_results_response and search_results_response.results: + search_results = search_results_response.results return { "profile": { - "static": response.profile.static, - "dynamic": response.profile.dynamic, + "static": profile.static if profile is not None else [], + "dynamic": profile.dynamic if profile is not None else [], }, "search_results": search_results, } diff --git a/packages/pipecat-sdk-python/tests/test_empty_profile.py b/packages/pipecat-sdk-python/tests/test_empty_profile.py new file mode 100644 index 000000000..ec3ccd261 --- /dev/null +++ b/packages/pipecat-sdk-python/tests/test_empty_profile.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import sys +import types +import unittest +from types import SimpleNamespace +from unittest.mock import AsyncMock + + +def _install_test_stubs() -> None: + if "loguru" not in sys.modules: + loguru_module = types.ModuleType("loguru") + + class _Logger: + def warning(self, *_args, **_kwargs): + return None + + def error(self, *_args, **_kwargs): + return None + + loguru_module.logger = _Logger() + sys.modules["loguru"] = loguru_module + + if "pydantic" not in sys.modules: + pydantic_module = types.ModuleType("pydantic") + + class BaseModel: + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + def Field(*, default=None, **_kwargs): + return default + + pydantic_module.BaseModel = BaseModel + pydantic_module.Field = Field + sys.modules["pydantic"] = pydantic_module + + if "pipecat" not in sys.modules: + pipecat_module = types.ModuleType("pipecat") + sys.modules["pipecat"] = pipecat_module + + frames_module = types.ModuleType("pipecat.frames.frames") + + class Frame: # pragma: no cover - import stub + pass + + class InputAudioRawFrame: # pragma: no cover - import stub + pass + + class LLMContextFrame: # pragma: no cover - import stub + pass + + class LLMMessagesFrame: # pragma: no cover - import stub + pass + + frames_module.Frame = Frame + frames_module.InputAudioRawFrame = InputAudioRawFrame + frames_module.LLMContextFrame = LLMContextFrame + frames_module.LLMMessagesFrame = LLMMessagesFrame + + llm_context_module = types.ModuleType("pipecat.processors.aggregators.llm_context") + + class LLMContext: # pragma: no cover - import stub + pass + + llm_context_module.LLMContext = LLMContext + + openai_context_module = types.ModuleType( + "pipecat.processors.aggregators.openai_llm_context" + ) + + class OpenAILLMContextFrame: # pragma: no cover - import stub + pass + + openai_context_module.OpenAILLMContextFrame = OpenAILLMContextFrame + + frame_processor_module = types.ModuleType("pipecat.processors.frame_processor") + + class FrameDirection: # pragma: no cover - import stub + pass + + class FrameProcessor: + def __init__(self, *args, **kwargs): + return None + + frame_processor_module.FrameDirection = FrameDirection + frame_processor_module.FrameProcessor = FrameProcessor + + sys.modules["pipecat.frames.frames"] = frames_module + sys.modules["pipecat.processors.aggregators.llm_context"] = llm_context_module + sys.modules[ + "pipecat.processors.aggregators.openai_llm_context" + ] = openai_context_module + sys.modules["pipecat.processors.frame_processor"] = frame_processor_module + + +_install_test_stubs() + +from supermemory_pipecat.service import SupermemoryPipecatService + + +class _MockSupermemoryClient: + def __init__(self, response): + self.profile = AsyncMock(return_value=response) + + +class TestSupermemoryPipecatNullProfile(unittest.IsolatedAsyncioTestCase): + async def test_retrieve_memories_handles_null_profile(self) -> None: + service = SupermemoryPipecatService(api_key="mock_key", user_id="new_user_123") + + response = SimpleNamespace(profile=None, search_results=None) + service._supermemory_client = _MockSupermemoryClient(response) + + result = await service._retrieve_memories("Hello world") + + self.assertEqual( + result, + { + "profile": {"static": [], "dynamic": []}, + "search_results": [], + }, + ) \ No newline at end of file