From bb74d12c13e7928f6ae4e58d1081b72c870c60c5 Mon Sep 17 00:00:00 2001 From: Zack Asofsky Date: Sun, 31 May 2026 14:48:03 -0400 Subject: [PATCH 1/3] Add golden validation and MemoryProfiler summary-page metrics. Extend the snapshot exporter to compute and validate the Unity Memory Profiler "Summary" page numbers, and add supporting snapshot decoding, reporting, and batch tooling. Summary metrics - SummaryMetricsCalculator replicates EntriesMemoryMap (build/sort/post-process), GetPointType classification, and the AllMemory/Managed summary builder heuristics (legacy untracked fallback, graphics estimation, map-resolved VM-root reassignment). - Export the Allocated Memory Distribution and Managed Heap Utilization breakdowns plus totals to a new summary_metrics table (DuckDB + SQLite). Golden validation - GoldenValidationRunner compares an exported DB against golden JSON (AssetBundle/SerializedFile/Remapper plus all summary categories), with committed/resident tolerances and a CLI `validate` command. - Add the com.unity.memory-snapshot-data-tools Unity package, whose editor GoldenValueExtractor reads golden values straight from the Memory Profiler's own summary model builders. - scripts/validate-golden.sh exports + validates in one step for CI. Snapshot decoding fixes/additions - Correct ProfileTarget_Info/_MemoryStats entry IDs (59/60, not 64/65) so TargetMemoryStats decodes; read SystemMemoryRegions_Type as ushort so Mapped/Device regions classify correctly. - Decode managed heap section types, target memory stats, and per-type static field bytes; crawl static-field roots so statically reachable managed objects are discovered. - Add SnapMetadataReader, SnapProfileTargetInfoParser, CaptureMetadata, ResidentMemoryCalculator, and MemoryMapResidentAggregator. Reports, batch export, CLI, docs - Multi-snapshot HTML report (builder, renderer, session grouping). - BatchExportRunner/ExportRunner and batch-export CLI command. - Document the .snap binary format (docs/snap-file-format.md); update the report skill and mkdocs nav. - Tests for golden validation, summary metrics, resident memory, batch export, and session grouping. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../skills/memory-snapshot-report/SKILL.md | 41 +- Cli/CliOptions.cs | 11 + Cli/CommandLineBuilder.cs | 206 ++++- Cli/Program.cs | 145 ++-- Core/Export/BatchExportRunner.cs | 169 ++++ Core/Export/ExportPipeline.cs | 16 + Core/Export/ExportRunner.cs | 125 +++ .../DuckDbExportDestination.cs | 151 +++- .../IExportDestinationWriter.cs | 8 + .../SqliteExportDestination.cs | 4 + Core/ExportDestination/SqliteWriter.cs | 196 ++++- Core/Models/CaptureMetadata.cs | 76 ++ Core/Models/ExportPipeline.cs | 19 + Core/Models/SnapshotData.cs | 25 +- Core/Models/SnapshotRows.cs | 47 +- Core/Models/SummaryMetrics.cs | 71 ++ Core/Parser/ManagedSnapshotCrawler.cs | 92 +++ Core/Parser/MemoryMapResidentAggregator.cs | 158 ++++ Core/Parser/ResidentMemoryCalculator.cs | 212 +++++ Core/Parser/SnapDataModel.cs | 103 ++- Core/Parser/SnapMetadataReader.cs | 117 +++ Core/Parser/SnapProfileTargetInfoParser.cs | 143 ++++ Core/Parser/SnapReader.cs | 52 ++ Core/Parser/SnapSectionDecoders.cs | 247 ++++++ Core/Parser/SnapshotBridge.cs | 88 ++- Core/Parser/SummaryMetricsCalculator.cs | 556 +++++++++++++ .../MultiSnapshotHtmlRenderer.cs | 315 ++++++++ .../MultiSnapshotReportBuilder.cs | 422 ++++++++++ .../MultiSnapshotReportModel.cs | 122 +++ .../MultiSnapshotReportRunner.cs | 85 ++ .../MultiSnapshotSessionGrouper.cs | 124 +++ .../MultiSnapshotSessionKey.cs | 150 ++++ .../MultiSnapshotReport/PlatformIconHtml.cs | 36 + Core/Report/ReportHtmlHelper.cs | 85 +- Core/Validation/DbScalarReader.cs | 72 ++ Core/Validation/GoldenValidationModels.cs | 162 ++++ Core/Validation/GoldenValidationQueries.cs | 55 ++ Core/Validation/GoldenValidationRunner.cs | 396 ++++++++++ Tests/BatchExportRunnerTests.cs | 62 ++ Tests/GoldenValidationRunnerTests.cs | 220 ++++++ Tests/MultiSnapshotSessionKeyTests.cs | 55 ++ Tests/ResidentMemoryCalculatorTests.cs | 112 +++ Tests/SnapshotBridgeTests.cs | 4 + Tests/SummaryMetricsCalculatorTests.cs | 94 +++ .../Editor.meta | 8 + .../Editor/GoldenValueExtractor.cs | 737 ++++++++++++++++++ .../Editor/GoldenValueExtractor.cs.meta | 2 + .../Editor/MemoryProfilerSnapshotLoader.cs | 166 ++++ .../MemoryProfilerSnapshotLoader.cs.meta | 2 + ...SnapshotDataTools.Validation.Editor.asmdef | 7 + ...hotDataTools.Validation.Editor.asmdef.meta | 7 + .../Editor/MemorySnapshotValidationHelpers.cs | 38 + .../MemorySnapshotValidationHelpers.cs.meta | 2 + .../Runtime.meta | 8 + .../MemorySnapshotDataTools.Validation.asmdef | 7 + ...rySnapshotDataTools.Validation.asmdef.meta | 7 + .../Runtime/ValidationGoldenData.cs | 103 +++ .../Runtime/ValidationGoldenData.cs.meta | 2 + .../package.json | 8 + .../package.json.meta | 7 + docs/snap-file-format.md | 533 +++++++++++++ mkdocs.yml | 1 + scripts/validate-golden.sh | 84 ++ 63 files changed, 7238 insertions(+), 140 deletions(-) create mode 100644 Core/Export/BatchExportRunner.cs create mode 100644 Core/Export/ExportRunner.cs create mode 100644 Core/Models/CaptureMetadata.cs create mode 100644 Core/Models/SummaryMetrics.cs create mode 100644 Core/Parser/MemoryMapResidentAggregator.cs create mode 100644 Core/Parser/ResidentMemoryCalculator.cs create mode 100644 Core/Parser/SnapMetadataReader.cs create mode 100644 Core/Parser/SnapProfileTargetInfoParser.cs create mode 100644 Core/Parser/SummaryMetricsCalculator.cs create mode 100644 Core/Report/MultiSnapshotReport/MultiSnapshotHtmlRenderer.cs create mode 100644 Core/Report/MultiSnapshotReport/MultiSnapshotReportBuilder.cs create mode 100644 Core/Report/MultiSnapshotReport/MultiSnapshotReportModel.cs create mode 100644 Core/Report/MultiSnapshotReport/MultiSnapshotReportRunner.cs create mode 100644 Core/Report/MultiSnapshotReport/MultiSnapshotSessionGrouper.cs create mode 100644 Core/Report/MultiSnapshotReport/MultiSnapshotSessionKey.cs create mode 100644 Core/Report/MultiSnapshotReport/PlatformIconHtml.cs create mode 100644 Core/Validation/DbScalarReader.cs create mode 100644 Core/Validation/GoldenValidationModels.cs create mode 100644 Core/Validation/GoldenValidationQueries.cs create mode 100644 Core/Validation/GoldenValidationRunner.cs create mode 100644 Tests/BatchExportRunnerTests.cs create mode 100644 Tests/GoldenValidationRunnerTests.cs create mode 100644 Tests/MultiSnapshotSessionKeyTests.cs create mode 100644 Tests/ResidentMemoryCalculatorTests.cs create mode 100644 Tests/SummaryMetricsCalculatorTests.cs create mode 100644 UnityPackage/com.unity.memory-snapshot-data-tools/Editor.meta create mode 100644 UnityPackage/com.unity.memory-snapshot-data-tools/Editor/GoldenValueExtractor.cs create mode 100644 UnityPackage/com.unity.memory-snapshot-data-tools/Editor/GoldenValueExtractor.cs.meta create mode 100644 UnityPackage/com.unity.memory-snapshot-data-tools/Editor/MemoryProfilerSnapshotLoader.cs create mode 100644 UnityPackage/com.unity.memory-snapshot-data-tools/Editor/MemoryProfilerSnapshotLoader.cs.meta create mode 100644 UnityPackage/com.unity.memory-snapshot-data-tools/Editor/MemorySnapshotDataTools.Validation.Editor.asmdef create mode 100644 UnityPackage/com.unity.memory-snapshot-data-tools/Editor/MemorySnapshotDataTools.Validation.Editor.asmdef.meta create mode 100644 UnityPackage/com.unity.memory-snapshot-data-tools/Editor/MemorySnapshotValidationHelpers.cs create mode 100644 UnityPackage/com.unity.memory-snapshot-data-tools/Editor/MemorySnapshotValidationHelpers.cs.meta create mode 100644 UnityPackage/com.unity.memory-snapshot-data-tools/Runtime.meta create mode 100644 UnityPackage/com.unity.memory-snapshot-data-tools/Runtime/MemorySnapshotDataTools.Validation.asmdef create mode 100644 UnityPackage/com.unity.memory-snapshot-data-tools/Runtime/MemorySnapshotDataTools.Validation.asmdef.meta create mode 100644 UnityPackage/com.unity.memory-snapshot-data-tools/Runtime/ValidationGoldenData.cs create mode 100644 UnityPackage/com.unity.memory-snapshot-data-tools/Runtime/ValidationGoldenData.cs.meta create mode 100644 UnityPackage/com.unity.memory-snapshot-data-tools/package.json create mode 100644 UnityPackage/com.unity.memory-snapshot-data-tools/package.json.meta create mode 100644 docs/snap-file-format.md create mode 100755 scripts/validate-golden.sh diff --git a/.cursor/skills/memory-snapshot-report/SKILL.md b/.cursor/skills/memory-snapshot-report/SKILL.md index d6dfd37..26f6719 100644 --- a/.cursor/skills/memory-snapshot-report/SKILL.md +++ b/.cursor/skills/memory-snapshot-report/SKILL.md @@ -30,7 +30,42 @@ dotnet run --project Cli/MemorySnapshotDataTools.Cli.csproj -- export .duckdb` alongside each file: + +```bash +dotnet run --project Cli/MemorySnapshotDataTools.Cli.csproj -- batch-export \ + --filter MyGame \ + --skip-existing \ + --verbose +``` + +- `--filter` is optional (case-insensitive substring on filenames). +- `--skip-existing` skips when the output DB is newer than the snap. +- Exit code `0` = all succeeded, `1` = one or more failures, `2` = cancelled. + +### 2. Validate export against Unity golden JSON + +The golden extractor lives in the `com.unity.memory-snapshot-data-tools` package under `UnityPackage/` +in this repo, imported into U6TestBed via a local `file:` path in `Packages/manifest.json`. After extracting +`*_golden.json` in Unity (**Tools → Memory Snapshot Validation → Extract Golden Values**): + +```bash +dotnet run --project Cli/MemorySnapshotDataTools.Cli.csproj -- validate \ + \ + +``` + +- Compares AssetBundle, SerializedFile (Unity Subsystems native roots), and Remapper metrics. +- Also compares the MemoryProfiler **Summary** page metrics from the `summary_metrics` table: + - *Allocated Memory Distribution*: Total Allocated, Total Resident, Native, Managed, + Executables & Mapped, Graphics (Estimated), Untracked. + - *Managed Heap Utilization*: Virtual Machine, Objects, Empty Heap Space. + - Committed bytes use a 1% / 64 KB tolerance (5% / 1 MB for the estimated Graphics and Untracked rows); + resident bytes use 1% / 64 KB and are skipped for Graphics and Untracked (resident unavailable). +- Writes `*_validation_result.json` next to the golden file unless `--out` is set. +- Exit code `0` = pass, `1` = metric mismatch, `3` = error. + +### 3. Generate HTML report ```bash dotnet run --project Cli/MemorySnapshotDataTools.Cli.csproj -- report --out report.html --verbose @@ -40,10 +75,10 @@ dotnet run --project Cli/MemorySnapshotDataTools.Cli.csproj -- report @@ -19,6 +22,14 @@ internal sealed class CliOptions public string ReportDbPath { get; set; } = string.Empty; public string? ReportOutputPath { get; set; } public string ReportTitle { get; set; } = "Memory Snapshot Report"; + public string BatchExportDirectory { get; set; } = string.Empty; + public string? BatchExportFilter { get; set; } + public bool SkipExisting { get; set; } + public bool ContinueOnError { get; set; } = true; + public string MultiReportDirectory { get; set; } = string.Empty; + public string? MultiReportFilter { get; set; } + public string GoldenPath { get; set; } = string.Empty; + public string? ValidationOutputPath { get; set; } public int BatchSize { get; set; } = 2048; public int QueueCapacity { get; set; } = 256; public ValidationMode Validate { get; set; } = ValidationMode.Minimal; diff --git a/Cli/CommandLineBuilder.cs b/Cli/CommandLineBuilder.cs index 5f0b9bf..a6e5141 100644 --- a/Cli/CommandLineBuilder.cs +++ b/Cli/CommandLineBuilder.cs @@ -8,7 +8,12 @@ namespace MemorySnapshotDataTools.Cli; /// internal static class CommandLineBuilder { - public static RootCommand Build(Func runExport, Func runReport) + public static RootCommand Build( + Func runExport, + Func runBatchExport, + Func runReport, + Func runMultiReport, + Func runValidateGolden) { var root = new RootCommand("Export Unity memory snapshots to DuckDB or SQLite and generate HTML reports."); @@ -54,11 +59,7 @@ public static RootCommand Build(Func runExport, Func { @@ -83,6 +84,90 @@ public static RootCommand Build(Func runExport, Func("directory") + { + Description = "Directory containing .snap files (top level only).", + Arity = ArgumentArity.ExactlyOne, + }; + batchExportCmd.Add(batchDirectoryArg); + + var batchFilterOpt = new Option("--filter") + { + Description = "Case-insensitive substring filter on snapshot filenames (e.g. MyGame).", + }; + var skipExistingOpt = new Option("--skip-existing") + { + Description = "Skip when the output database exists and is newer than the .snap file.", + }; + var continueOnErrorOpt = new Option("--continue-on-error") + { + Description = "Continue exporting after a single-file failure.", + DefaultValueFactory = _ => true, + }; + + var batchBatchSizeOpt = new Option("--batch-size") + { + Description = "Rows per produced batch.", + DefaultValueFactory = _ => 2048, + }; + var batchQueueCapacityOpt = new Option("--queue-capacity") + { + Description = "Max queued batches.", + DefaultValueFactory = _ => 256, + }; + var batchValidateOpt = new Option("--validate") + { + Description = "Validation mode: none, minimal, or full.", + DefaultValueFactory = _ => "minimal", + }; + batchValidateOpt.AcceptOnlyFromAmong("none", "minimal", "full"); + var batchDestinationOpt = new Option("--destination") + { + Description = "Export backend: duckdb or sqlite.", + DefaultValueFactory = _ => "duckdb", + }; + batchDestinationOpt.AcceptOnlyFromAmong("duckdb", "sqlite"); + var batchVerboseOpt = new Option("--verbose") + { + Description = "Print progress updates.", + }; + + batchExportCmd.Add(batchFilterOpt); + batchExportCmd.Add(skipExistingOpt); + batchExportCmd.Add(continueOnErrorOpt); + AddExportOptions(batchExportCmd, batchBatchSizeOpt, batchQueueCapacityOpt, batchValidateOpt, batchDestinationOpt, batchVerboseOpt); + + batchExportCmd.SetAction((ParseResult parseResult) => + { + var directory = ExpandPath(parseResult.GetValue(batchDirectoryArg)!); + if (!Directory.Exists(directory)) + { + Console.Error.WriteLine($"Directory not found: {directory}"); + return 1; + } + + var options = new CliOptions + { + Command = CommandKind.BatchExport, + BatchExportDirectory = directory, + BatchExportFilter = parseResult.GetValue(batchFilterOpt), + SkipExisting = parseResult.GetValue(skipExistingOpt), + ContinueOnError = parseResult.GetValue(continueOnErrorOpt), + BatchSize = parseResult.GetValue(batchBatchSizeOpt), + QueueCapacity = parseResult.GetValue(batchQueueCapacityOpt), + Validate = ParseValidationMode(parseResult.GetValue(batchValidateOpt)!), + Destination = parseResult.GetValue(batchDestinationOpt)!.ToLowerInvariant() == "sqlite" + ? DestinationKind.Sqlite + : DestinationKind.DuckDb, + Verbose = parseResult.GetValue(batchVerboseOpt), + }; + return runBatchExport(options); + }); + // ---- report ---- var reportCmd = new Command("report", "Generate an HTML report from an exported database."); var databaseArg = new Argument("database") @@ -130,11 +215,120 @@ public static RootCommand Build(Func runExport, Func("directory") + { + Description = "Directory containing .duckdb or .db snapshot databases.", + Arity = ArgumentArity.ExactlyOne, + }; + multiReportCmd.Add(directoryArg); + + var filterOpt = new Option("--filter") + { + Description = "Case-insensitive substring filter on database filenames (e.g. MyGame).", + }; + var multiOutOpt = new Option("--out") + { + Description = "Output HTML file path (default: temp file + open in browser).", + }; + var multiTitleOpt = new Option("--title") + { + Description = "Report title.", + DefaultValueFactory = _ => "Multi-Snapshot Memory Report", + }; + var multiVerboseOpt = new Option("--verbose") + { + Description = "Print progress and timings.", + }; + + multiReportCmd.Add(filterOpt); + multiReportCmd.Add(multiOutOpt); + multiReportCmd.Add(multiTitleOpt); + multiReportCmd.Add(multiVerboseOpt); + + multiReportCmd.SetAction((ParseResult parseResult) => + { + var directory = ExpandPath(parseResult.GetValue(directoryArg)!); + if (!Directory.Exists(directory)) + { + Console.Error.WriteLine($"Directory not found: {directory}"); + return 1; + } + + var outPath = parseResult.GetValue(multiOutOpt); + var options = new CliOptions + { + Command = CommandKind.MultiReport, + MultiReportDirectory = directory, + MultiReportFilter = parseResult.GetValue(filterOpt), + ReportOutputPath = string.IsNullOrWhiteSpace(outPath) ? null : ExpandPath(outPath!), + ReportTitle = parseResult.GetValue(multiTitleOpt)!, + Verbose = parseResult.GetValue(multiVerboseOpt), + }; + return runMultiReport(options); + }); + + // ---- validate ---- + var validateCmd = new Command( + "validate", + "Compare an exported database against a Unity golden JSON file."); + var goldenArg = new Argument("golden") + { + Description = "Path to *_golden.json from Unity GoldenValueExtractor.", + Arity = ArgumentArity.ExactlyOne, + }; + var validateDatabaseArg = new Argument("database") + { + Description = "Path to the exported .duckdb or .db file.", + Arity = ArgumentArity.ExactlyOne, + }; + validateCmd.Add(goldenArg); + validateCmd.Add(validateDatabaseArg); + + var validateOutOpt = new Option("--out") + { + Description = "Output validation result JSON path (default: next to golden file).", + }; + validateCmd.Add(validateOutOpt); + + validateCmd.SetAction((ParseResult parseResult) => + { + var goldenPath = ExpandPath(parseResult.GetValue(goldenArg)!); + var databasePath = ExpandPath(parseResult.GetValue(validateDatabaseArg)!); + var options = new CliOptions + { + Command = CommandKind.ValidateGolden, + GoldenPath = goldenPath, + ReportDbPath = databasePath, + ValidationOutputPath = parseResult.GetValue(validateOutOpt), + }; + return runValidateGolden(options); + }); + root.Add(exportCmd); + root.Add(batchExportCmd); root.Add(reportCmd); + root.Add(multiReportCmd); + root.Add(validateCmd); return root; } + private static void AddExportOptions( + Command command, + Option batchSizeOpt, + Option queueCapacityOpt, + Option validateOpt, + Option destinationOpt, + Option verboseOpt) + { + command.Add(batchSizeOpt); + command.Add(queueCapacityOpt); + command.Add(validateOpt); + command.Add(destinationOpt); + command.Add(verboseOpt); + } + private static ValidationMode ParseValidationMode(string value) { return value.ToLowerInvariant() switch diff --git a/Cli/Program.cs b/Cli/Program.cs index 86fdc75..bb68308 100644 --- a/Cli/Program.cs +++ b/Cli/Program.cs @@ -1,9 +1,8 @@ -using System.Diagnostics; using MemorySnapshotDataTools; using MemorySnapshotDataTools.Export; -using MemorySnapshotDataTools.ExportDestination; -using MemorySnapshotDataTools.Parser; using MemorySnapshotDataTools.Report; +using MemorySnapshotDataTools.Report.MultiSnapshotReport; +using MemorySnapshotDataTools.Validation; namespace MemorySnapshotDataTools.Cli; @@ -11,78 +10,63 @@ internal static class Program { private static int Main(string[] args) { - var root = CommandLineBuilder.Build(RunExport, RunReport); + var root = CommandLineBuilder.Build(RunExport, RunBatchExport, RunReport, RunMultiReport, RunValidateGolden); return root.Parse(args).Invoke(); } private static int RunExport(CliOptions options) { - var destination = ExportDestinationFactory.Create(options.Destination); var progress = new ConsoleProgress(options.Verbose); - progress.Report($"Backend: {destination.DestinationName}", force: true); - - using var cts = new CancellationTokenSource(); - Console.CancelKeyPress += (_, e) => - { - e.Cancel = true; - cts.Cancel(); - }; + using var cts = CreateCancellationSource(); try { - var sw = Stopwatch.StartNew(); - - var exportOptions = new ExportRunOptions - { - OutputDbPath = options.OutputDbPath, - BatchSize = options.BatchSize, - QueueCapacity = options.QueueCapacity, - Validate = options.Validate, - }; - - var extractSw = Stopwatch.StartNew(); - var rawData = RunStage("snapshot-extract", progress, () => SnapshotBridge.ExtractRawData(options.SnapshotPath, progress, cts.Token)); - extractSw.Stop(); - - var pipelineSw = Stopwatch.StartNew(); - var counts = RunStage("pipeline-write", progress, () => ExportPipeline.Run(exportOptions, rawData, destination, progress, cts.Token)); - pipelineSw.Stop(); - - var validationSw = Stopwatch.StartNew(); - RunStage("validation", progress, () => destination.Validate(options.OutputDbPath, rawData, options.Validate)); - validationSw.Stop(); - - counts.TotalMs = sw.ElapsedMilliseconds; - var pipelineRps = pipelineSw.ElapsedMilliseconds > 0 - ? rawData.TotalRows * 1000.0 / pipelineSw.ElapsedMilliseconds - : 0.0; - - progress.Report( - $"Done. backend={destination.DestinationName}, native_objects={counts.NativeObjects}, managed_objects={counts.ManagedObjects}, connections={counts.Connections}, native_roots={counts.NativeRoots}, " + - $"memory_regions={counts.MemoryRegions}, native_allocations={counts.NativeAllocations}, " + - $"extract_ms={extractSw.ElapsedMilliseconds}, pipeline_ms={pipelineSw.ElapsedMilliseconds}, validation_ms={validationSw.ElapsedMilliseconds}, total_ms={counts.TotalMs}, " + - $"pipeline_rps={pipelineRps:N0}, backend_insert_ms={counts.BackendInsertMs}, backend_commit_ms={counts.BackendCommitMs}, backend_index_ms={counts.BackendIndexBuildMs}, " + - $"insert_ms_by_table(native={counts.NativeObjectInsertMs}, managed={counts.ManagedObjectInsertMs}, connections={counts.ConnectionInsertMs}, roots={counts.NativeRootInsertMs}, regions={counts.MemoryRegionInsertMs}, allocations={counts.NativeAllocationInsertMs})"); - return 0; + return ExportRunner.Run( + options.SnapshotPath, + options.OutputDbPath, + new ExportRunOptions + { + BatchSize = options.BatchSize, + QueueCapacity = options.QueueCapacity, + Validate = options.Validate, + }, + options.Destination, + progress, + cts.Token); } catch (OperationCanceledException) { Console.Error.WriteLine("Export cancelled."); return 2; } - catch (Exception ex) + } + + private static int RunBatchExport(CliOptions options) + { + var progress = new ConsoleProgress(options.Verbose); + using var cts = CreateCancellationSource(); + + try { - Console.Error.WriteLine("Export failed."); - if (ex is ExportStageException stageEx) - { - Console.Error.WriteLine($"Failure stage: {stageEx.Stage}"); - Console.Error.WriteLine(stageEx.InnerException ?? stageEx); - } - else - { - Console.Error.WriteLine(ex); - } - return 3; + return BatchExportRunner.Run( + new BatchExportRunOptions + { + Directory = options.BatchExportDirectory, + NameFilter = options.BatchExportFilter, + Destination = options.Destination, + BatchSize = options.BatchSize, + QueueCapacity = options.QueueCapacity, + Validate = options.Validate, + SkipExisting = options.SkipExisting, + ContinueOnError = options.ContinueOnError, + }, + progress, + cts.Token); + } + catch (OperationCanceledException) + { + Console.Error.WriteLine("Batch export cancelled."); + return 2; } } @@ -98,29 +82,44 @@ private static int RunReport(CliOptions options) return ReportRunner.Run(reportOptions, progress); } - private static void RunStage(string stage, ConsoleProgress progress, Action action) + private static int RunMultiReport(CliOptions options) { - progress.Report($"[{stage}] start", force: true); - try - { - action(); - } - catch (Exception ex) when (ex is not ExportStageException) + var multiOptions = new MultiSnapshotReportRunOptions { - throw new ExportStageException(stage, ex); - } + Directory = options.MultiReportDirectory, + NameFilter = options.MultiReportFilter, + ReportOutputPath = options.ReportOutputPath, + ReportTitle = options.ReportTitle, + }; + var progress = new ConsoleProgress(options.Verbose); + return MultiSnapshotReportRunner.Run(multiOptions, progress); } - private static T RunStage(string stage, ConsoleProgress progress, Func action) + private static int RunValidateGolden(CliOptions options) { - progress.Report($"[{stage}] start", force: true); try { - return action(); + return GoldenValidationRunner.ValidateAndWriteResult( + options.GoldenPath, + options.ReportDbPath, + options.ValidationOutputPath); } - catch (Exception ex) when (ex is not ExportStageException) + catch (Exception ex) { - throw new ExportStageException(stage, ex); + Console.Error.WriteLine("Golden validation failed."); + Console.Error.WriteLine(ex.Message); + return 3; } } + + private static CancellationTokenSource CreateCancellationSource() + { + var cts = new CancellationTokenSource(); + Console.CancelKeyPress += (_, e) => + { + e.Cancel = true; + cts.Cancel(); + }; + return cts; + } } diff --git a/Core/Export/BatchExportRunner.cs b/Core/Export/BatchExportRunner.cs new file mode 100644 index 0000000..18b4b13 --- /dev/null +++ b/Core/Export/BatchExportRunner.cs @@ -0,0 +1,169 @@ +namespace MemorySnapshotDataTools.Export; + +/// +/// Options for exporting every .snap file in a directory to a database alongside it. +/// +public sealed class BatchExportRunOptions +{ + /// Directory to scan for .snap files (top level only). + public string Directory { get; set; } = string.Empty; + + /// Optional case-insensitive substring filter on snapshot filenames. + public string? NameFilter { get; set; } + + /// Database backend (default DuckDB). + public DestinationKind Destination { get; set; } = DestinationKind.DuckDb; + + /// Pipeline batch size passed to each export. + public int BatchSize { get; set; } = 2048; + + /// Pipeline queue capacity passed to each export. + public int QueueCapacity { get; set; } = 256; + + /// Post-export validation mode. + public ValidationMode Validate { get; set; } = ValidationMode.Minimal; + + /// When true, skip snapshots whose output database exists and is newer than the snap file. + public bool SkipExisting { get; set; } + + /// When true, continue exporting remaining files after a single-file failure. + public bool ContinueOnError { get; set; } = true; +} + +/// +/// Result summary for a run. +/// +public sealed class BatchExportResult +{ + /// Snapshot files that exported successfully. + public IReadOnlyList Succeeded { get; init; } = []; + + /// Snapshot files skipped because output was up to date. + public IReadOnlyList Skipped { get; init; } = []; + + /// Snapshot paths paired with failure messages. + public IReadOnlyList<(string SnapshotPath, string Error)> Failed { get; init; } = []; +} + +/// +/// Exports multiple .snap files from a directory to .duckdb or .db files with matching basenames. +/// +public static class BatchExportRunner +{ + /// + /// Discovers .snap files in matching an optional name filter. + /// + public static IReadOnlyList DiscoverSnapshotFiles(string directory, string? nameFilter) + { + if (!Directory.Exists(directory)) + return []; + + return Directory.EnumerateFiles(directory, "*.snap", SearchOption.TopDirectoryOnly) + .Where(p => string.IsNullOrWhiteSpace(nameFilter) + || Path.GetFileName(p).Contains(nameFilter, StringComparison.OrdinalIgnoreCase)) + .OrderBy(p => p, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + /// + /// Exports each discovered snapshot. Returns process exit code: 0 all ok, 1 partial failure, 2 cancelled, 3 fatal. + /// + public static int Run(BatchExportRunOptions options, IProgressReporter progress, CancellationToken token = default) + { + var directory = Path.GetFullPath(options.Directory); + if (!Directory.Exists(directory)) + { + Console.Error.WriteLine($"Directory not found: {directory}"); + return 3; + } + + var snapshots = DiscoverSnapshotFiles(directory, options.NameFilter); + if (snapshots.Count == 0) + { + Console.Error.WriteLine($"No .snap files found in {directory}" + + (string.IsNullOrWhiteSpace(options.NameFilter) ? "." : $" matching filter '{options.NameFilter}'.")); + return 3; + } + + var extension = options.Destination == DestinationKind.Sqlite ? ".db" : ".duckdb"; + var pipelineOptions = new ExportRunOptions + { + BatchSize = options.BatchSize, + QueueCapacity = options.QueueCapacity, + Validate = options.Validate, + }; + + progress.Report( + $"Batch export: {directory} ({snapshots.Count} snapshots, filter: {options.NameFilter ?? "(none)"}, destination: {options.Destination})", + force: true); + + var succeeded = new List(); + var skipped = new List(); + var failed = new List<(string, string)>(); + + for (var i = 0; i < snapshots.Count; i++) + { + token.ThrowIfCancellationRequested(); + + var snapPath = snapshots[i]; + var outputPath = Path.ChangeExtension(snapPath, extension); + var label = $"[{i + 1}/{snapshots.Count}] {Path.GetFileName(snapPath)}"; + progress.Report($"=== {label} ===", force: true); + + if (options.SkipExisting && ShouldSkip(snapPath, outputPath)) + { + progress.Report($"Skipped (up-to-date): {Path.GetFileName(outputPath)}", force: true); + skipped.Add(snapPath); + continue; + } + + var exitCode = ExportRunner.Run( + snapPath, + outputPath, + pipelineOptions, + options.Destination, + progress, + token); + + if (exitCode == 0) + { + succeeded.Add(snapPath); + continue; + } + + if (exitCode == 2) + return 2; + + failed.Add((snapPath, $"export exited with code {exitCode}")); + if (!options.ContinueOnError) + break; + } + + var result = new BatchExportResult + { + Succeeded = succeeded, + Skipped = skipped, + Failed = failed, + }; + + progress.Report( + $"Batch complete: {result.Succeeded.Count} succeeded, {result.Skipped.Count} skipped, {result.Failed.Count} failed.", + force: true); + + foreach (var (snapPath, error) in result.Failed) + Console.Error.WriteLine($" FAILED {Path.GetFileName(snapPath)}: {error}"); + + if (result.Failed.Count > 0) + return 1; + + return 0; + } + + private static bool ShouldSkip(string snapPath, string outputPath) + { + if (!File.Exists(outputPath)) + return false; + + return File.GetLastWriteTimeUtc(outputPath) >= File.GetLastWriteTimeUtc(snapPath); + } +} diff --git a/Core/Export/ExportPipeline.cs b/Core/Export/ExportPipeline.cs index cc01aa0..4864d4f 100644 --- a/Core/Export/ExportPipeline.cs +++ b/Core/Export/ExportPipeline.cs @@ -49,6 +49,7 @@ public static ExportCounts Run(ExportRunOptions options, RawSnapshotData rawData Task.Run(() => ProduceNativeRoots(rawData.NativeRoots, queue, state, options.BatchSize, cts.Token), cts.Token), Task.Run(() => ProduceMemoryRegions(rawData.MemoryRegions, queue, state, options.BatchSize, cts.Token), cts.Token), Task.Run(() => ProduceNativeAllocations(rawData.NativeAllocations, queue, state, options.BatchSize, cts.Token), cts.Token), + Task.Run(() => ProduceSystemMemoryRegions(rawData.SystemMemoryRegions, queue, state, options.BatchSize, cts.Token), cts.Token), Task.Run(() => ProduceNativeObjects(rawData.NativeObjects, queue, state, options.BatchSize, cts.Token), cts.Token), Task.Run(() => ProduceManagedObjects(rawData.ManagedObjects, queue, state, options.BatchSize, cts.Token), cts.Token), Task.Run(() => ProduceConnections(rawData.Connections, queue, state, options.BatchSize, cts.Token), cts.Token), @@ -68,6 +69,7 @@ public static ExportCounts Run(ExportRunOptions options, RawSnapshotData rawData counts.NativeRoots = rawData.NativeRoots.Count; counts.MemoryRegions = rawData.MemoryRegions.Count; counts.NativeAllocations = rawData.NativeAllocations.Count; + counts.SystemMemoryRegions = rawData.SystemMemoryRegions.Count; counts.MaterializeMs = materializeSw.ElapsedMilliseconds; counts.WriteMs = writeSw.ElapsedMilliseconds; counts.BackendInsertMs = writeStats.TotalInsertMs; @@ -79,6 +81,7 @@ public static ExportCounts Run(ExportRunOptions options, RawSnapshotData rawData counts.NativeRootInsertMs = writeStats.NativeRootInsertMs; counts.MemoryRegionInsertMs = writeStats.MemoryRegionInsertMs; counts.NativeAllocationInsertMs = writeStats.NativeAllocationInsertMs; + counts.SystemMemoryRegionInsertMs = writeStats.SystemMemoryRegionInsertMs; if (state.MaterializedRows != rawData.TotalRows) throw new InvalidOperationException($"Materialized rows mismatch. expected={rawData.TotalRows}, actual={state.MaterializedRows}"); @@ -191,6 +194,19 @@ private static void ProduceNativeAllocations(List rows, Blo }); } + private static void ProduceSystemMemoryRegions(List rows, BlockingCollection queue, PipelineState state, int batchSize, CancellationToken token) + { + ProduceBatches(rows.Count, batchSize, token, start => + { + var end = Math.Min(start + batchSize, rows.Count); + var buffer = new SystemMemoryRegionRow[end - start]; + rows.CopyTo(start, buffer, 0, buffer.Length); + queue.Add(WriteBatch.ForSystemMemoryRegions(buffer), token); + state.IncrementQueuedBatches(); + state.AddMaterialized(buffer.Length); + }); + } + private static void ProduceManagedObjects(List rows, BlockingCollection queue, PipelineState state, int batchSize, CancellationToken token) { ProduceBatches(rows.Count, batchSize, token, start => diff --git a/Core/Export/ExportRunner.cs b/Core/Export/ExportRunner.cs new file mode 100644 index 0000000..807b030 --- /dev/null +++ b/Core/Export/ExportRunner.cs @@ -0,0 +1,125 @@ +using System.Diagnostics; +using MemorySnapshotDataTools.ExportDestination; +using MemorySnapshotDataTools.Parser; + +namespace MemorySnapshotDataTools.Export; + +/// +/// Runs a single snapshot export: extract, write database, validate. +/// +public static class ExportRunner +{ + /// + /// Exports one to . + /// + /// Path to the .snap file. + /// Output database path (.duckdb or .db). + /// Pipeline batch/validation options. + /// DuckDB or SQLite backend. + /// Progress reporter. + /// Cancellation token. + /// Process exit code: 0 success, 2 cancelled, 3 error. + public static int Run( + string snapshotPath, + string outputDbPath, + ExportRunOptions pipelineOptions, + DestinationKind destination, + IProgressReporter progress, + CancellationToken token = default) + { + var destinationWriter = ExportDestinationFactory.Create(destination); + progress.Report($"Backend: {destinationWriter.DestinationName}", force: true); + + try + { + var sw = Stopwatch.StartNew(); + + var exportOptions = new ExportRunOptions + { + OutputDbPath = outputDbPath, + BatchSize = pipelineOptions.BatchSize, + QueueCapacity = pipelineOptions.QueueCapacity, + Validate = pipelineOptions.Validate, + }; + + var extractSw = Stopwatch.StartNew(); + var rawData = RunStage("snapshot-extract", progress, () => + SnapshotBridge.ExtractRawData(snapshotPath, progress, token)); + extractSw.Stop(); + token.ThrowIfCancellationRequested(); + + var pipelineSw = Stopwatch.StartNew(); + var counts = RunStage("pipeline-write", progress, () => + ExportPipeline.Run(exportOptions, rawData, destinationWriter, progress, token)); + pipelineSw.Stop(); + token.ThrowIfCancellationRequested(); + + RunStage("summary-metrics-write", progress, () => + destinationWriter.WriteSummaryMetrics(outputDbPath, rawData.SummaryMetrics)); + + var validationSw = Stopwatch.StartNew(); + RunStage("validation", progress, () => + destinationWriter.Validate(outputDbPath, rawData, pipelineOptions.Validate)); + validationSw.Stop(); + + counts.TotalMs = sw.ElapsedMilliseconds; + var pipelineRps = pipelineSw.ElapsedMilliseconds > 0 + ? rawData.TotalRows * 1000.0 / pipelineSw.ElapsedMilliseconds + : 0.0; + + progress.Report( + $"Done. backend={destinationWriter.DestinationName}, native_objects={counts.NativeObjects}, managed_objects={counts.ManagedObjects}, connections={counts.Connections}, native_roots={counts.NativeRoots}, " + + $"memory_regions={counts.MemoryRegions}, native_allocations={counts.NativeAllocations}, system_memory_regions={counts.SystemMemoryRegions}, " + + $"extract_ms={extractSw.ElapsedMilliseconds}, pipeline_ms={pipelineSw.ElapsedMilliseconds}, validation_ms={validationSw.ElapsedMilliseconds}, total_ms={counts.TotalMs}, " + + $"pipeline_rps={pipelineRps:N0}, backend_insert_ms={counts.BackendInsertMs}, backend_commit_ms={counts.BackendCommitMs}, backend_index_ms={counts.BackendIndexBuildMs}, " + + $"insert_ms_by_table(native={counts.NativeObjectInsertMs}, managed={counts.ManagedObjectInsertMs}, connections={counts.ConnectionInsertMs}, roots={counts.NativeRootInsertMs}, regions={counts.MemoryRegionInsertMs}, allocations={counts.NativeAllocationInsertMs})"); + return 0; + } + catch (OperationCanceledException) + { + progress.Report("Export cancelled.", force: true); + return 2; + } + catch (Exception ex) + { + progress.Report("Export failed.", force: true); + if (ex is ExportStageException stageEx) + { + progress.Report($"Failure stage: {stageEx.Stage}", force: true); + progress.Report((stageEx.InnerException ?? stageEx).ToString(), force: true); + } + else + { + progress.Report(ex.ToString(), force: true); + } + + return 3; + } + } + + private static void RunStage(string stage, IProgressReporter progress, Action action) + { + progress.Report($"[{stage}] start", force: true); + try + { + action(); + } + catch (Exception ex) when (ex is not ExportStageException) + { + throw new ExportStageException(stage, ex); + } + } + + private static T RunStage(string stage, IProgressReporter progress, Func action) + { + progress.Report($"[{stage}] start", force: true); + try + { + return action(); + } + catch (Exception ex) when (ex is not ExportStageException) + { + throw new ExportStageException(stage, ex); + } + } +} diff --git a/Core/ExportDestination/DuckDbExportDestination.cs b/Core/ExportDestination/DuckDbExportDestination.cs index 84661d1..dd7dccc 100644 --- a/Core/ExportDestination/DuckDbExportDestination.cs +++ b/Core/ExportDestination/DuckDbExportDestination.cs @@ -47,10 +47,23 @@ public WriteStats ConsumeAndWrite( // Insert snapshot_info using positional parameters (DuckDB uses ? placeholders) using (var cmd = connection.CreateCommand()) { - cmd.CommandText = "INSERT INTO snapshot_info(snapshot_path, exported_at_utc, unity_version) VALUES (?, ?, ?);"; + cmd.CommandText = """ + INSERT INTO snapshot_info( + snapshot_path, exported_at_utc, unity_version, + snap_format_version, session_guid, product_name, platform, record_date_utc) + VALUES (?, ?, ?, ?, ?, ?, ?, ?); + """; cmd.Parameters.Add(new DuckDBParameter { Value = snapshotInfo.SnapshotPath }); cmd.Parameters.Add(new DuckDBParameter { Value = snapshotInfo.ExportedAtUtc }); cmd.Parameters.Add(new DuckDBParameter { Value = snapshotInfo.UnityVersion ?? (object)DBNull.Value }); + cmd.Parameters.Add(new DuckDBParameter { Value = snapshotInfo.SnapFormatVersion == 0 ? (object)DBNull.Value : snapshotInfo.SnapFormatVersion }); + cmd.Parameters.Add(new DuckDBParameter + { + Value = snapshotInfo.SessionGuid == 0 ? (object)DBNull.Value : unchecked((long)snapshotInfo.SessionGuid), + }); + cmd.Parameters.Add(new DuckDBParameter { Value = string.IsNullOrEmpty(snapshotInfo.ProductName) ? (object)DBNull.Value : snapshotInfo.ProductName }); + cmd.Parameters.Add(new DuckDBParameter { Value = string.IsNullOrEmpty(snapshotInfo.Platform) ? (object)DBNull.Value : snapshotInfo.Platform }); + cmd.Parameters.Add(new DuckDBParameter { Value = string.IsNullOrEmpty(snapshotInfo.RecordDateUtc) ? (object)DBNull.Value : snapshotInfo.RecordDateUtc }); cmd.ExecuteNonQuery(); } state.AddWritten(1); @@ -64,6 +77,7 @@ public WriteStats ConsumeAndWrite( using (var rootAppender = connection.CreateAppender("native_roots")) using (var regionAppender = connection.CreateAppender("memory_regions")) using (var allocationAppender = connection.CreateAppender("native_allocations")) + using (var systemRegionAppender = connection.CreateAppender("system_memory_regions")) { foreach (var batch in queue.GetConsumingEnumerable(token)) { @@ -76,15 +90,21 @@ public WriteStats ConsumeAndWrite( foreach (var row in batch.NativeObjects) { // INTEGER columns get int, BIGINT columns get long (type must match exactly) - nativeAppender.CreateRow() - .AppendValue(row.NativeObjectIndex) // int → INTEGER - .AppendValue(row.InstanceId ?? string.Empty) // string → VARCHAR - .AppendValue(row.Name ?? string.Empty) // string → VARCHAR - .AppendValue(unchecked((long)row.SizeBytes)) // ulong → BIGINT - .AppendValue(row.TypeIndex) // int → INTEGER - .AppendValue(row.NativeTypeName ?? string.Empty) // string → VARCHAR - .AppendValue(row.IsDestroyed) // bool → BOOLEAN - .EndRow(); + var nativeRow = nativeAppender.CreateRow() + .AppendValue(row.NativeObjectIndex) + .AppendValue(row.InstanceId ?? string.Empty) + .AppendValue(row.Name ?? string.Empty) + .AppendValue(unchecked((long)row.SizeBytes)) + .AppendValue(unchecked((long)row.NativeObjectAddress)) + .AppendValue(row.RootReferenceId) + .AppendValue(row.TypeIndex) + .AppendValue(row.NativeTypeName ?? string.Empty) + .AppendValue(row.IsDestroyed); + if (row.ResidentSizeBytes.HasValue) + nativeRow.AppendValue(unchecked((long)row.ResidentSizeBytes.Value)); + else + nativeRow.AppendNullValue(); + nativeRow.EndRow(); } nativeSw.Stop(); stats.NativeObjectRows += batch.NativeObjects.Length; @@ -136,13 +156,17 @@ public WriteStats ConsumeAndWrite( var rootSw = Stopwatch.StartNew(); foreach (var row in batch.NativeRoots) { - rootAppender.CreateRow() - .AppendValue(row.RootIndex) // int → INTEGER - .AppendValue(row.RootId) // long → BIGINT - .AppendValue(row.AreaName ?? string.Empty) // VARCHAR - .AppendValue(row.ObjectName ?? string.Empty) // VARCHAR - .AppendValue(unchecked((long)row.AccumulatedSizeBytes)) // ulong → BIGINT - .EndRow(); + var rootRow = rootAppender.CreateRow() + .AppendValue(row.RootIndex) + .AppendValue(row.RootId) + .AppendValue(row.AreaName ?? string.Empty) + .AppendValue(row.ObjectName ?? string.Empty) + .AppendValue(unchecked((long)row.AccumulatedSizeBytes)); + if (row.ResidentSizeBytes.HasValue) + rootRow.AppendValue(unchecked((long)row.ResidentSizeBytes.Value)); + else + rootRow.AppendNullValue(); + rootRow.EndRow(); } rootSw.Stop(); stats.NativeRootRows += batch.NativeRoots.Length; @@ -186,7 +210,11 @@ public WriteStats ConsumeAndWrite( .AppendValue(unchecked((long)row.OverheadSizeBytes)) // ulong → BIGINT .AppendValue(unchecked((long)row.PaddingSizeBytes)); // ulong → BIGINT if (row.MemoryRegionIndex >= 0) - r.AppendValue(row.MemoryRegionIndex); // int → INTEGER + r.AppendValue(row.MemoryRegionIndex); + else + r.AppendNullValue(); + if (row.RootReferenceId >= 0) + r.AppendValue(row.RootReferenceId); else r.AppendNullValue(); r.EndRow(); @@ -196,6 +224,25 @@ public WriteStats ConsumeAndWrite( stats.NativeAllocationInsertMs += allocSw.ElapsedMilliseconds; state.AddWritten(batch.NativeAllocations.Length); break; + + case WriteBatchKind.SystemMemoryRegions: + var sysSw = Stopwatch.StartNew(); + foreach (var row in batch.SystemMemoryRegions) + { + systemRegionAppender.CreateRow() + .AppendValue(row.RegionIndex) + .AppendValue(unchecked((long)row.Address)) + .AppendValue(unchecked((long)row.SizeBytes)) + .AppendValue(unchecked((long)row.ResidentBytes)) + .AppendValue(row.Type) + .AppendValue(row.Name ?? string.Empty) + .EndRow(); + } + sysSw.Stop(); + stats.SystemMemoryRegionRows += batch.SystemMemoryRegions.Length; + stats.SystemMemoryRegionInsertMs += sysSw.ElapsedMilliseconds; + state.AddWritten(batch.SystemMemoryRegions.Length); + break; } } } // appenders disposed (flushed + committed) here @@ -215,6 +262,29 @@ public WriteStats ConsumeAndWrite( #endregion + #region SummaryMetrics + + /// + public void WriteSummaryMetrics(string dbPath, SummaryMetrics metrics) + { + using var connection = new DuckDBConnection($"Data Source={dbPath}"); + connection.Open(); + + using var appender = connection.CreateAppender("summary_metrics"); + foreach (var (group, category, committed, resident, residentAvailable) in SummaryMetricsTable.Enumerate(metrics)) + { + appender.CreateRow() + .AppendValue(group) + .AppendValue(category) + .AppendValue(unchecked((long)committed)) + .AppendValue(unchecked((long)resident)) + .AppendValue(residentAvailable ? 1 : 0) + .EndRow(); + } + } + + #endregion + #region Validation /// @@ -232,21 +302,25 @@ public void Validate(string dbPath, RawSnapshotData rawData, ValidationMode mode var rootCount = QueryCount(connection, "SELECT COUNT(*) FROM native_roots;"); var regionCount = QueryCount(connection, "SELECT COUNT(*) FROM memory_regions;"); var allocationCount = QueryCount(connection, "SELECT COUNT(*) FROM native_allocations;"); + var systemRegionCount = QueryCount(connection, "SELECT COUNT(*) FROM system_memory_regions;"); if (nativeCount != rawData.NativeObjects.Count || managedCount != rawData.ManagedObjects.Count || connectionCount != rawData.Connections.Count || rootCount != rawData.NativeRoots.Count || regionCount != rawData.MemoryRegions.Count || - allocationCount != rawData.NativeAllocations.Count) + allocationCount != rawData.NativeAllocations.Count || + systemRegionCount != rawData.SystemMemoryRegions.Count) { throw new InvalidOperationException( $"DuckDB validation count mismatch. " + $"expected=(native={rawData.NativeObjects.Count}, managed={rawData.ManagedObjects.Count}, " + $"connections={rawData.Connections.Count}, roots={rawData.NativeRoots.Count}, " + - $"regions={rawData.MemoryRegions.Count}, allocations={rawData.NativeAllocations.Count}) " + + $"regions={rawData.MemoryRegions.Count}, allocations={rawData.NativeAllocations.Count}, " + + $"system_regions={rawData.SystemMemoryRegions.Count}) " + $"actual=(native={nativeCount}, managed={managedCount}, connections={connectionCount}, " + - $"roots={rootCount}, regions={regionCount}, allocations={allocationCount})"); + $"roots={rootCount}, regions={regionCount}, allocations={allocationCount}, " + + $"system_regions={systemRegionCount})"); } if (mode == ValidationMode.Full) @@ -353,7 +427,12 @@ private static long QueryCount(DuckDBConnection connection, string sql) CREATE OR REPLACE TABLE snapshot_info ( snapshot_path VARCHAR NOT NULL, exported_at_utc VARCHAR NOT NULL, - unity_version VARCHAR + unity_version VARCHAR, + snap_format_version INTEGER, + session_guid BIGINT, + product_name VARCHAR, + platform VARCHAR, + record_date_utc VARCHAR ); CREATE OR REPLACE TABLE native_objects ( @@ -361,9 +440,12 @@ CREATE OR REPLACE TABLE native_objects ( instance_id VARCHAR, name VARCHAR, size_bytes BIGINT NOT NULL, + native_object_address BIGINT NOT NULL DEFAULT 0, + root_reference_id BIGINT NOT NULL DEFAULT -1, type_index INTEGER, native_type_name VARCHAR, - is_destroyed BOOLEAN NOT NULL + is_destroyed BOOLEAN NOT NULL, + resident_size_bytes BIGINT ); CREATE OR REPLACE TABLE managed_objects ( @@ -388,7 +470,8 @@ CREATE OR REPLACE TABLE native_roots ( root_id BIGINT NOT NULL, area_name VARCHAR, object_name VARCHAR, - accumulated_size_bytes BIGINT NOT NULL + accumulated_size_bytes BIGINT NOT NULL, + resident_size_bytes BIGINT ); CREATE OR REPLACE TABLE memory_regions ( @@ -407,7 +490,25 @@ CREATE OR REPLACE TABLE native_allocations ( size_bytes BIGINT NOT NULL, overhead_size_bytes BIGINT NOT NULL, padding_size_bytes BIGINT NOT NULL, - memory_region_index INTEGER + memory_region_index INTEGER, + root_reference_id BIGINT +); + +CREATE OR REPLACE TABLE system_memory_regions ( + region_index INTEGER PRIMARY KEY, + address BIGINT NOT NULL, + size_bytes BIGINT NOT NULL, + resident_bytes BIGINT NOT NULL, + type INTEGER NOT NULL, + name VARCHAR +); + +CREATE OR REPLACE TABLE summary_metrics ( + metric_group VARCHAR NOT NULL, + category VARCHAR NOT NULL, + committed_bytes BIGINT NOT NULL, + resident_bytes BIGINT NOT NULL, + resident_available INTEGER NOT NULL ); """; diff --git a/Core/ExportDestination/IExportDestinationWriter.cs b/Core/ExportDestination/IExportDestinationWriter.cs index 37dec7e..6926c37 100644 --- a/Core/ExportDestination/IExportDestinationWriter.cs +++ b/Core/ExportDestination/IExportDestinationWriter.cs @@ -28,6 +28,14 @@ WriteStats ConsumeAndWrite( PipelineState state, CancellationToken token); + /// + /// Writes the MemoryProfiler summary metrics into the summary_metrics table. Called after the + /// main pipeline write (which creates the schema) and before validation. + /// + /// Output database file path. + /// Summary metrics computed during extraction. + void WriteSummaryMetrics(string dbPath, SummaryMetrics metrics); + /// /// Runs optional validation on the written database (e.g. row count checks, referential integrity) according to . /// diff --git a/Core/ExportDestination/SqliteExportDestination.cs b/Core/ExportDestination/SqliteExportDestination.cs index dfa2b64..8933d6c 100644 --- a/Core/ExportDestination/SqliteExportDestination.cs +++ b/Core/ExportDestination/SqliteExportDestination.cs @@ -20,6 +20,10 @@ public WriteStats ConsumeAndWrite( CancellationToken token) => SqliteWriter.ConsumeAndWrite(dbPath, snapshotInfo, queue, state, token); + /// + public void WriteSummaryMetrics(string dbPath, SummaryMetrics metrics) + => SqliteWriter.WriteSummaryMetrics(dbPath, metrics); + /// public void Validate(string dbPath, RawSnapshotData rawData, ValidationMode mode) => SqliteWriter.Validate(dbPath, rawData, mode); diff --git a/Core/ExportDestination/SqliteWriter.cs b/Core/ExportDestination/SqliteWriter.cs index 7a01266..659e8f4 100644 --- a/Core/ExportDestination/SqliteWriter.cs +++ b/Core/ExportDestination/SqliteWriter.cs @@ -40,13 +40,15 @@ public static void Validate(string dbPath, RawSnapshotData rawData, ValidationMo var rootCount = QueryCount(connection, "SELECT COUNT(*) FROM native_roots;"); var regionCount = QueryCount(connection, "SELECT COUNT(*) FROM memory_regions;"); var allocationCount = QueryCount(connection, "SELECT COUNT(*) FROM native_allocations;"); + var systemRegionCount = QueryCount(connection, "SELECT COUNT(*) FROM system_memory_regions;"); if (nativeCount != rawData.NativeObjects.Count || managedCount != rawData.ManagedObjects.Count || connectionCount != rawData.Connections.Count || rootCount != rawData.NativeRoots.Count || regionCount != rawData.MemoryRegions.Count || - allocationCount != rawData.NativeAllocations.Count) + allocationCount != rawData.NativeAllocations.Count || + systemRegionCount != rawData.SystemMemoryRegions.Count) { throw new InvalidOperationException("SQLite validation count mismatch between extracted rows and persisted rows."); } @@ -135,6 +137,44 @@ FROM native_allocations a #endregion + #region SummaryMetrics + + /// + /// Inserts the MemoryProfiler summary metrics into the summary_metrics table (created by the schema script). + /// + public static void WriteSummaryMetrics(string dbPath, SummaryMetrics metrics) + { + using var connection = new SqliteConnection($"Data Source={dbPath}"); + connection.Open(); + + using var transaction = connection.BeginTransaction(); + using var cmd = connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.CommandText = """ + INSERT INTO summary_metrics(metric_group, category, committed_bytes, resident_bytes, resident_available) + VALUES ($g, $c, $cb, $rb, $ra); + """; + var g = cmd.Parameters.Add("$g", Microsoft.Data.Sqlite.SqliteType.Text); + var c = cmd.Parameters.Add("$c", Microsoft.Data.Sqlite.SqliteType.Text); + var cb = cmd.Parameters.Add("$cb", Microsoft.Data.Sqlite.SqliteType.Integer); + var rb = cmd.Parameters.Add("$rb", Microsoft.Data.Sqlite.SqliteType.Integer); + var ra = cmd.Parameters.Add("$ra", Microsoft.Data.Sqlite.SqliteType.Integer); + + foreach (var (group, category, committed, resident, residentAvailable) in SummaryMetricsTable.Enumerate(metrics)) + { + g.Value = group; + c.Value = category; + cb.Value = unchecked((long)committed); + rb.Value = unchecked((long)resident); + ra.Value = residentAvailable ? 1 : 0; + cmd.ExecuteNonQuery(); + } + + transaction.Commit(); + } + + #endregion + #region ConsumeAndWrite /// @@ -174,10 +214,20 @@ public static WriteStats ConsumeAndWrite( using var snapshotCmd = connection.CreateCommand(); snapshotCmd.Transaction = transaction; - snapshotCmd.CommandText = "INSERT INTO snapshot_info(snapshot_path, exported_at_utc, unity_version) VALUES ($p, $e, $u);"; + snapshotCmd.CommandText = """ + INSERT INTO snapshot_info( + snapshot_path, exported_at_utc, unity_version, + snap_format_version, session_guid, product_name, platform, record_date_utc) + VALUES ($p, $e, $u, $sf, $sg, $pn, $pl, $rd); + """; snapshotCmd.Parameters.AddWithValue("$p", snapshotInfo.SnapshotPath); snapshotCmd.Parameters.AddWithValue("$e", snapshotInfo.ExportedAtUtc); snapshotCmd.Parameters.AddWithValue("$u", snapshotInfo.UnityVersion); + snapshotCmd.Parameters.AddWithValue("$sf", snapshotInfo.SnapFormatVersion == 0 ? DBNull.Value : snapshotInfo.SnapFormatVersion); + snapshotCmd.Parameters.AddWithValue("$sg", snapshotInfo.SessionGuid == 0 ? DBNull.Value : unchecked((long)snapshotInfo.SessionGuid)); + snapshotCmd.Parameters.AddWithValue("$pn", string.IsNullOrEmpty(snapshotInfo.ProductName) ? DBNull.Value : snapshotInfo.ProductName); + snapshotCmd.Parameters.AddWithValue("$pl", string.IsNullOrEmpty(snapshotInfo.Platform) ? DBNull.Value : snapshotInfo.Platform); + snapshotCmd.Parameters.AddWithValue("$rd", string.IsNullOrEmpty(snapshotInfo.RecordDateUtc) ? DBNull.Value : snapshotInfo.RecordDateUtc); snapshotCmd.ExecuteNonQuery(); state.AddWritten(1); var insertSw = Stopwatch.StartNew(); @@ -187,6 +237,7 @@ public static WriteStats ConsumeAndWrite( using var rootCmd = PrepareRootInsert(connection, transaction); using var regionCmd = PrepareRegionInsert(connection, transaction); using var allocationCmd = PrepareAllocationInsert(connection, transaction); + using var systemRegionCmd = PrepareSystemRegionInsert(connection, transaction); foreach (var batch in queue.GetConsumingEnumerable(token)) { @@ -202,9 +253,14 @@ public static WriteStats ConsumeAndWrite( nativeCmd.Parameters[1].Value = row.InstanceId ?? string.Empty; nativeCmd.Parameters[2].Value = row.Name ?? string.Empty; nativeCmd.Parameters[3].Value = unchecked((long)row.SizeBytes); - nativeCmd.Parameters[4].Value = row.TypeIndex; - nativeCmd.Parameters[5].Value = row.NativeTypeName ?? string.Empty; - nativeCmd.Parameters[6].Value = row.IsDestroyed ? 1 : 0; + nativeCmd.Parameters[4].Value = unchecked((long)row.NativeObjectAddress); + nativeCmd.Parameters[5].Value = row.RootReferenceId; + nativeCmd.Parameters[6].Value = row.TypeIndex; + nativeCmd.Parameters[7].Value = row.NativeTypeName ?? string.Empty; + nativeCmd.Parameters[8].Value = row.IsDestroyed ? 1 : 0; + nativeCmd.Parameters[9].Value = row.ResidentSizeBytes.HasValue + ? unchecked((long)row.ResidentSizeBytes.Value) + : DBNull.Value; nativeCmd.ExecuteNonQuery(); } nativeSw.Stop(); @@ -257,6 +313,9 @@ public static WriteStats ConsumeAndWrite( rootCmd.Parameters[2].Value = row.AreaName ?? string.Empty; rootCmd.Parameters[3].Value = row.ObjectName ?? string.Empty; rootCmd.Parameters[4].Value = unchecked((long)row.AccumulatedSizeBytes); + rootCmd.Parameters[5].Value = row.ResidentSizeBytes.HasValue + ? unchecked((long)row.ResidentSizeBytes.Value) + : DBNull.Value; rootCmd.ExecuteNonQuery(); } rootSw.Stop(); @@ -294,6 +353,7 @@ public static WriteStats ConsumeAndWrite( allocationCmd.Parameters[3].Value = unchecked((long)row.OverheadSizeBytes); allocationCmd.Parameters[4].Value = unchecked((long)row.PaddingSizeBytes); allocationCmd.Parameters[5].Value = row.MemoryRegionIndex >= 0 ? row.MemoryRegionIndex : DBNull.Value; + allocationCmd.Parameters[6].Value = row.RootReferenceId >= 0 ? row.RootReferenceId : DBNull.Value; allocationCmd.ExecuteNonQuery(); } allocationSw.Stop(); @@ -301,6 +361,24 @@ public static WriteStats ConsumeAndWrite( stats.NativeAllocationInsertMs += allocationSw.ElapsedMilliseconds; state.AddWritten(batch.NativeAllocations.Length); break; + + case WriteBatchKind.SystemMemoryRegions: + var sysSw = Stopwatch.StartNew(); + foreach (var row in batch.SystemMemoryRegions) + { + systemRegionCmd.Parameters[0].Value = row.RegionIndex; + systemRegionCmd.Parameters[1].Value = unchecked((long)row.Address); + systemRegionCmd.Parameters[2].Value = unchecked((long)row.SizeBytes); + systemRegionCmd.Parameters[3].Value = unchecked((long)row.ResidentBytes); + systemRegionCmd.Parameters[4].Value = row.Type; + systemRegionCmd.Parameters[5].Value = row.Name ?? string.Empty; + systemRegionCmd.ExecuteNonQuery(); + } + sysSw.Stop(); + stats.SystemMemoryRegionRows += batch.SystemMemoryRegions.Length; + stats.SystemMemoryRegionInsertMs += sysSw.ElapsedMilliseconds; + state.AddWritten(batch.SystemMemoryRegions.Length); + break; } } insertSw.Stop(); @@ -349,14 +427,17 @@ private static SqliteCommand PrepareNativeInsert(SqliteConnection connection, Sq { var command = connection.CreateCommand(); command.Transaction = tx; - command.CommandText = "INSERT INTO native_objects(native_object_index, instance_id, name, size_bytes, type_index, native_type_name, is_destroyed) VALUES ($i, $id, $n, $s, $t, $tn, $d);"; + command.CommandText = "INSERT INTO native_objects(native_object_index, instance_id, name, size_bytes, native_object_address, root_reference_id, type_index, native_type_name, is_destroyed, resident_size_bytes) VALUES ($i, $id, $n, $s, $addr, $rid, $t, $tn, $d, $r);"; _ = command.Parameters.Add("$i", SqliteType.Integer); _ = command.Parameters.Add("$id", SqliteType.Text); _ = command.Parameters.Add("$n", SqliteType.Text); _ = command.Parameters.Add("$s", SqliteType.Integer); + _ = command.Parameters.Add("$addr", SqliteType.Integer); + _ = command.Parameters.Add("$rid", SqliteType.Integer); _ = command.Parameters.Add("$t", SqliteType.Integer); _ = command.Parameters.Add("$tn", SqliteType.Text); _ = command.Parameters.Add("$d", SqliteType.Integer); + _ = command.Parameters.Add("$r", SqliteType.Integer); return command; } @@ -391,12 +472,13 @@ private static SqliteCommand PrepareRootInsert(SqliteConnection connection, Sqli { var command = connection.CreateCommand(); command.Transaction = tx; - command.CommandText = "INSERT INTO native_roots(root_index, root_id, area_name, object_name, accumulated_size_bytes) VALUES ($i, $rid, $a, $o, $s);"; + command.CommandText = "INSERT INTO native_roots(root_index, root_id, area_name, object_name, accumulated_size_bytes, resident_size_bytes) VALUES ($i, $rid, $a, $o, $s, $r);"; _ = command.Parameters.Add("$i", SqliteType.Integer); _ = command.Parameters.Add("$rid", SqliteType.Integer); _ = command.Parameters.Add("$a", SqliteType.Text); _ = command.Parameters.Add("$o", SqliteType.Text); _ = command.Parameters.Add("$s", SqliteType.Integer); + _ = command.Parameters.Add("$r", SqliteType.Integer); return command; } @@ -419,13 +501,28 @@ private static SqliteCommand PrepareAllocationInsert(SqliteConnection connection { var command = connection.CreateCommand(); command.Transaction = tx; - command.CommandText = "INSERT INTO native_allocations(allocation_index, address, size_bytes, overhead_size_bytes, padding_size_bytes, memory_region_index) VALUES ($i, $a, $s, $o, $p, $r);"; + command.CommandText = "INSERT INTO native_allocations(allocation_index, address, size_bytes, overhead_size_bytes, padding_size_bytes, memory_region_index, root_reference_id) VALUES ($i, $a, $s, $o, $p, $mr, $rid);"; _ = command.Parameters.Add("$i", SqliteType.Integer); _ = command.Parameters.Add("$a", SqliteType.Integer); _ = command.Parameters.Add("$s", SqliteType.Integer); _ = command.Parameters.Add("$o", SqliteType.Integer); _ = command.Parameters.Add("$p", SqliteType.Integer); + _ = command.Parameters.Add("$mr", SqliteType.Integer); + _ = command.Parameters.Add("$rid", SqliteType.Integer); + return command; + } + + private static SqliteCommand PrepareSystemRegionInsert(SqliteConnection connection, SqliteTransaction tx) + { + var command = connection.CreateCommand(); + command.Transaction = tx; + command.CommandText = "INSERT INTO system_memory_regions(region_index, address, size_bytes, resident_bytes, type, name) VALUES ($i, $a, $s, $r, $t, $n);"; + _ = command.Parameters.Add("$i", SqliteType.Integer); + _ = command.Parameters.Add("$a", SqliteType.Integer); + _ = command.Parameters.Add("$s", SqliteType.Integer); _ = command.Parameters.Add("$r", SqliteType.Integer); + _ = command.Parameters.Add("$t", SqliteType.Integer); + _ = command.Parameters.Add("$n", SqliteType.Text); return command; } @@ -465,8 +562,8 @@ private static SqliteCommand CreateBulkInsertCommand( private static void WriteNativeObjectRows(SqliteConnection connection, SqliteTransaction tx, NativeObjectRow[] rows) { - const int cols = 7; - const string insertPrefix = "INSERT INTO native_objects(native_object_index, instance_id, name, size_bytes, type_index, native_type_name, is_destroyed) VALUES "; + const int cols = 10; + const string insertPrefix = "INSERT INTO native_objects(native_object_index, instance_id, name, size_bytes, native_object_address, root_reference_id, type_index, native_type_name, is_destroyed, resident_size_bytes) VALUES "; var rowsPerStatement = RowsPerStatement(cols); for (var start = 0; start < rows.Length; start += rowsPerStatement) { @@ -480,9 +577,12 @@ private static void WriteNativeObjectRows(SqliteConnection connection, SqliteTra command.Parameters.AddWithValue($"$p{p + 1}", row.InstanceId ?? string.Empty); command.Parameters.AddWithValue($"$p{p + 2}", row.Name ?? string.Empty); command.Parameters.AddWithValue($"$p{p + 3}", unchecked((long)row.SizeBytes)); - command.Parameters.AddWithValue($"$p{p + 4}", row.TypeIndex); - command.Parameters.AddWithValue($"$p{p + 5}", row.NativeTypeName ?? string.Empty); - command.Parameters.AddWithValue($"$p{p + 6}", row.IsDestroyed ? 1 : 0); + command.Parameters.AddWithValue($"$p{p + 4}", unchecked((long)row.NativeObjectAddress)); + command.Parameters.AddWithValue($"$p{p + 5}", row.RootReferenceId); + command.Parameters.AddWithValue($"$p{p + 6}", row.TypeIndex); + command.Parameters.AddWithValue($"$p{p + 7}", row.NativeTypeName ?? string.Empty); + command.Parameters.AddWithValue($"$p{p + 8}", row.IsDestroyed ? 1 : 0); + command.Parameters.AddWithValue($"$p{p + 9}", row.ResidentSizeBytes.HasValue ? unchecked((long)row.ResidentSizeBytes.Value) : DBNull.Value); } command.ExecuteNonQuery(); } @@ -537,8 +637,8 @@ private static void WriteConnectionRows(SqliteConnection connection, SqliteTrans private static void WriteNativeRootRows(SqliteConnection connection, SqliteTransaction tx, NativeRootRow[] rows) { - const int cols = 5; - const string insertPrefix = "INSERT INTO native_roots(root_index, root_id, area_name, object_name, accumulated_size_bytes) VALUES "; + const int cols = 6; + const string insertPrefix = "INSERT INTO native_roots(root_index, root_id, area_name, object_name, accumulated_size_bytes, resident_size_bytes) VALUES "; var rowsPerStatement = RowsPerStatement(cols); for (var start = 0; start < rows.Length; start += rowsPerStatement) { @@ -553,6 +653,7 @@ private static void WriteNativeRootRows(SqliteConnection connection, SqliteTrans command.Parameters.AddWithValue($"$p{p + 2}", row.AreaName ?? string.Empty); command.Parameters.AddWithValue($"$p{p + 3}", row.ObjectName ?? string.Empty); command.Parameters.AddWithValue($"$p{p + 4}", unchecked((long)row.AccumulatedSizeBytes)); + command.Parameters.AddWithValue($"$p{p + 5}", row.ResidentSizeBytes.HasValue ? unchecked((long)row.ResidentSizeBytes.Value) : DBNull.Value); } command.ExecuteNonQuery(); } @@ -585,8 +686,8 @@ private static void WriteMemoryRegionRows(SqliteConnection connection, SqliteTra private static void WriteNativeAllocationRows(SqliteConnection connection, SqliteTransaction tx, NativeAllocationRow[] rows) { - const int cols = 6; - const string insertPrefix = "INSERT INTO native_allocations(allocation_index, address, size_bytes, overhead_size_bytes, padding_size_bytes, memory_region_index) VALUES "; + const int cols = 7; + const string insertPrefix = "INSERT INTO native_allocations(allocation_index, address, size_bytes, overhead_size_bytes, padding_size_bytes, memory_region_index, root_reference_id) VALUES "; var rowsPerStatement = RowsPerStatement(cols); for (var start = 0; start < rows.Length; start += rowsPerStatement) { @@ -602,6 +703,31 @@ private static void WriteNativeAllocationRows(SqliteConnection connection, Sqlit command.Parameters.AddWithValue($"$p{p + 3}", unchecked((long)row.OverheadSizeBytes)); command.Parameters.AddWithValue($"$p{p + 4}", unchecked((long)row.PaddingSizeBytes)); command.Parameters.AddWithValue($"$p{p + 5}", row.MemoryRegionIndex >= 0 ? row.MemoryRegionIndex : DBNull.Value); + command.Parameters.AddWithValue($"$p{p + 6}", row.RootReferenceId >= 0 ? row.RootReferenceId : DBNull.Value); + } + command.ExecuteNonQuery(); + } + } + + private static void WriteSystemMemoryRegionRows(SqliteConnection connection, SqliteTransaction tx, SystemMemoryRegionRow[] rows) + { + const int cols = 6; + const string insertPrefix = "INSERT INTO system_memory_regions(region_index, address, size_bytes, resident_bytes, type, name) VALUES "; + var rowsPerStatement = RowsPerStatement(cols); + for (var start = 0; start < rows.Length; start += rowsPerStatement) + { + var count = Math.Min(rowsPerStatement, rows.Length - start); + using var command = CreateBulkInsertCommand(connection, tx, insertPrefix, count, cols); + for (var i = 0; i < count; i++) + { + var row = rows[start + i]; + var p = i * cols; + command.Parameters.AddWithValue($"$p{p}", row.RegionIndex); + command.Parameters.AddWithValue($"$p{p + 1}", unchecked((long)row.Address)); + command.Parameters.AddWithValue($"$p{p + 2}", unchecked((long)row.SizeBytes)); + command.Parameters.AddWithValue($"$p{p + 3}", unchecked((long)row.ResidentBytes)); + command.Parameters.AddWithValue($"$p{p + 4}", row.Type); + command.Parameters.AddWithValue($"$p{p + 5}", row.Name ?? string.Empty); } command.ExecuteNonQuery(); } @@ -645,11 +771,17 @@ private static long QueryCount(SqliteConnection connection, string sql) DROP TABLE IF EXISTS native_roots; DROP TABLE IF EXISTS memory_regions; DROP TABLE IF EXISTS native_allocations; +DROP TABLE IF EXISTS system_memory_regions; CREATE TABLE snapshot_info ( snapshot_path TEXT NOT NULL, exported_at_utc TEXT NOT NULL, - unity_version TEXT + unity_version TEXT, + snap_format_version INTEGER, + session_guid INTEGER, + product_name TEXT, + platform TEXT, + record_date_utc TEXT ); CREATE TABLE native_objects ( @@ -657,9 +789,12 @@ CREATE TABLE native_objects ( instance_id TEXT, name TEXT, size_bytes INTEGER NOT NULL, + native_object_address INTEGER NOT NULL DEFAULT 0, + root_reference_id INTEGER NOT NULL DEFAULT -1, type_index INTEGER, native_type_name TEXT, - is_destroyed INTEGER NOT NULL DEFAULT 0 + is_destroyed INTEGER NOT NULL DEFAULT 0, + resident_size_bytes INTEGER ); CREATE TABLE managed_objects ( @@ -684,7 +819,8 @@ CREATE TABLE native_roots ( root_id INTEGER NOT NULL, area_name TEXT, object_name TEXT, - accumulated_size_bytes INTEGER NOT NULL + accumulated_size_bytes INTEGER NOT NULL, + resident_size_bytes INTEGER ); CREATE TABLE memory_regions ( @@ -703,7 +839,25 @@ CREATE TABLE native_allocations ( size_bytes INTEGER NOT NULL, overhead_size_bytes INTEGER NOT NULL, padding_size_bytes INTEGER NOT NULL, - memory_region_index INTEGER + memory_region_index INTEGER, + root_reference_id INTEGER +); + +CREATE TABLE system_memory_regions ( + region_index INTEGER PRIMARY KEY, + address INTEGER NOT NULL, + size_bytes INTEGER NOT NULL, + resident_bytes INTEGER NOT NULL, + type INTEGER NOT NULL, + name TEXT +); + +CREATE TABLE summary_metrics ( + metric_group TEXT NOT NULL, + category TEXT NOT NULL, + committed_bytes INTEGER NOT NULL, + resident_bytes INTEGER NOT NULL, + resident_available INTEGER NOT NULL ); """; diff --git a/Core/Models/CaptureMetadata.cs b/Core/Models/CaptureMetadata.cs new file mode 100644 index 0000000..b428a2b --- /dev/null +++ b/Core/Models/CaptureMetadata.cs @@ -0,0 +1,76 @@ +namespace MemorySnapshotDataTools; + +/// +/// Capture-time metadata from a Unity memory snapshot file (profile target, session, platform). +/// +public sealed record CaptureMetadata +{ + /// Invalid session GUID when not present in the snapshot. + public const uint InvalidSessionGuid = 0; + + /// Session identifier shared by snapshots taken in the same player run. + public uint SessionGuid { get; init; } + + /// Project or product name from the capture target. + public string ProductName { get; init; } = string.Empty; + + /// Unity editor/player version string (e.g. 6000.3.11f1). + public string UnityVersion { get; init; } = string.Empty; + + /// Runtime platform name (e.g. IPhonePlayer, Android). + public string Platform { get; init; } = string.Empty; + + /// Snapshot file format version from Metadata_Version. + public uint SnapFormatVersion { get; init; } + + /// Capture timestamp from metadata, if available. + public DateTime? RecordDateUtc { get; init; } + + /// Whether a non-zero session GUID was read from the file. + public bool HasProfilerSession => SessionGuid != InvalidSessionGuid; + + /// Normalized platform for UI (iOS, Android, or other). + public CapturePlatformKind PlatformKind => CapturePlatformKindExtensions.FromPlatformName(Platform); +} + +/// Coarse platform bucket for report icons and labels. +public enum CapturePlatformKind +{ + Unknown, + IOS, + Android, + Other, +} + +/// Maps Unity runtime platform strings to . +public static class CapturePlatformKindExtensions +{ + /// Classifies a platform string from snapshot metadata. + public static CapturePlatformKind FromPlatformName(string? platform) + { + if (string.IsNullOrWhiteSpace(platform)) + return CapturePlatformKind.Unknown; + + var p = platform.AsSpan().Trim(); + if (p.Contains("IPhone", StringComparison.OrdinalIgnoreCase) + || p.Contains("iOS", StringComparison.OrdinalIgnoreCase) + || p.Equals("tvOS", StringComparison.OrdinalIgnoreCase)) + { + return CapturePlatformKind.IOS; + } + + if (p.Contains("Android", StringComparison.OrdinalIgnoreCase)) + return CapturePlatformKind.Android; + + return CapturePlatformKind.Other; + } + + /// Short label for session headers. + public static string ToDisplayLabel(this CapturePlatformKind kind) => kind switch + { + CapturePlatformKind.IOS => "iOS", + CapturePlatformKind.Android => "Android", + CapturePlatformKind.Other => "Other", + _ => string.Empty, + }; +} diff --git a/Core/Models/ExportPipeline.cs b/Core/Models/ExportPipeline.cs index a735e97..4ca1ea2 100644 --- a/Core/Models/ExportPipeline.cs +++ b/Core/Models/ExportPipeline.cs @@ -13,6 +13,7 @@ public enum WriteBatchKind NativeRoots, MemoryRegions, NativeAllocations, + SystemMemoryRegions, } /// @@ -43,6 +44,9 @@ public sealed class WriteBatch /// Populated when is . public NativeAllocationRow[] NativeAllocations { get; init; } = []; + /// Populated when is . + public SystemMemoryRegionRow[] SystemMemoryRegions { get; init; } = []; + /// Creates a batch of native object rows. public static WriteBatch ForNativeObjects(NativeObjectRow[] rows) => new() { Kind = WriteBatchKind.NativeObjects, NativeObjects = rows }; @@ -60,6 +64,9 @@ public sealed class WriteBatch /// Creates a batch of native allocation rows. public static WriteBatch ForNativeAllocations(NativeAllocationRow[] rows) => new() { Kind = WriteBatchKind.NativeAllocations, NativeAllocations = rows }; + + /// Creates a batch of OS system memory region rows. + public static WriteBatch ForSystemMemoryRegions(SystemMemoryRegionRow[] rows) => new() { Kind = WriteBatchKind.SystemMemoryRegions, SystemMemoryRegions = rows }; } /// @@ -129,6 +136,9 @@ public sealed class ExportCounts /// Number of native allocations written. public int NativeAllocations; + /// Number of OS system memory regions written. + public int SystemMemoryRegions; + /// Time spent materializing batches (ms). public long MaterializeMs; @@ -164,6 +174,9 @@ public sealed class ExportCounts /// Per-table insert times (ms). public long NativeAllocationInsertMs; + + /// Per-table insert times (ms). + public long SystemMemoryRegionInsertMs; } /// @@ -190,6 +203,9 @@ public sealed class WriteStats /// Rows written per table. public long NativeAllocationRows; + /// Rows written per table. + public long SystemMemoryRegionRows; + /// Insert time per table (ms). public long NativeObjectInsertMs; @@ -208,6 +224,9 @@ public sealed class WriteStats /// Insert time per table (ms). public long NativeAllocationInsertMs; + /// Insert time per table (ms). + public long SystemMemoryRegionInsertMs; + /// Total time spent in inserts (ms). public long TotalInsertMs; diff --git a/Core/Models/SnapshotData.cs b/Core/Models/SnapshotData.cs index e7d334e..a5b66a0 100644 --- a/Core/Models/SnapshotData.cs +++ b/Core/Models/SnapshotData.cs @@ -27,9 +27,15 @@ public sealed class RawSnapshotData /// Allocations within native memory regions. public List NativeAllocations { get; } = []; + /// OS system memory regions (format v16+). + public List SystemMemoryRegions { get; } = []; + + /// MemoryProfiler "Summary" page metrics (Allocated Memory Distribution + Managed Heap Utilization). + public SummaryMetrics SummaryMetrics { get; set; } = new(); + /// Total number of data rows (all lists combined); used for pipeline progress. public long TotalRows => NativeObjects.Count + ManagedObjects.Count + Connections.Count - + NativeRoots.Count + MemoryRegions.Count + NativeAllocations.Count; + + NativeRoots.Count + MemoryRegions.Count + NativeAllocations.Count + SystemMemoryRegions.Count; } /// @@ -44,6 +50,21 @@ public sealed class SnapshotInfo /// When the snapshot was exported (UTC), as a string for display/storage. public string ExportedAtUtc { get; set; } = string.Empty; - /// Unity version or format string from the snapshot. + /// Unity version string from snapshot metadata, or format fallback. public string UnityVersion { get; set; } = string.Empty; + + /// Snap file format version (Metadata_Version). + public uint SnapFormatVersion { get; set; } + + /// Profiler session GUID from the capture target. + public uint SessionGuid { get; set; } + + /// Product or project name from the capture target. + public string ProductName { get; set; } = string.Empty; + + /// Runtime platform name (e.g. IPhonePlayer). + public string Platform { get; set; } = string.Empty; + + /// Capture timestamp (UTC ISO-8601), when known. + public string RecordDateUtc { get; set; } = string.Empty; } diff --git a/Core/Models/SnapshotRows.cs b/Core/Models/SnapshotRows.cs index fc0c577..a394921 100644 --- a/Core/Models/SnapshotRows.cs +++ b/Core/Models/SnapshotRows.cs @@ -14,9 +14,22 @@ public struct NativeObjectRow /// Display name. public string Name; - /// Size in bytes. + /// Size in bytes (allocated / committed). public ulong SizeBytes; + /// Native object base address from the snapshot. + public ulong NativeObjectAddress; + + /// Native root reference id linking this object to native_roots, or -1 when unknown. + public long RootReferenceId; + + /// + /// Resident size in bytes for the object's native root (allocations and object ranges under the same + /// ), matching Unity Memory Profiler processed-root totals; null when not + /// computable (format < 17) or when is unknown. + /// + public ulong? ResidentSizeBytes; + /// Index into the native type names array. public int TypeIndex; @@ -89,8 +102,11 @@ public struct NativeRootRow /// Object name for the root. public string ObjectName; - /// Accumulated size in bytes for this root. + /// Accumulated size in bytes for this root (allocated). public ulong AccumulatedSizeBytes; + + /// Resident size in bytes aggregated from rooted objects and allocations, or null when not computable. + public ulong? ResidentSizeBytes; } /// @@ -142,4 +158,31 @@ public struct NativeAllocationRow /// Containing memory region index, or -1. public int MemoryRegionIndex; + + /// Root reference ID linking to native_roots, or -1. + public long RootReferenceId; +} + +/// +/// One row from the system_memory_regions table: an OS-level memory region with committed and resident totals. +/// +public struct SystemMemoryRegionRow +{ + /// Zero-based region index. + public int RegionIndex; + + /// Region base address. + public ulong Address; + + /// Committed (allocated) size in bytes for the region. + public ulong SizeBytes; + + /// Resident size in bytes for the region. + public ulong ResidentBytes; + + /// Region type code from the snapshot. + public int Type; + + /// Region name. + public string Name; } diff --git a/Core/Models/SummaryMetrics.cs b/Core/Models/SummaryMetrics.cs new file mode 100644 index 0000000..ca2b96d --- /dev/null +++ b/Core/Models/SummaryMetrics.cs @@ -0,0 +1,71 @@ +namespace MemorySnapshotDataTools; + +/// +/// MemoryProfiler "Summary" page metrics computed from a snapshot: the Allocated Memory Distribution +/// and Managed Heap Utilization breakdowns plus overall totals. Mirrors what the Unity Memory Profiler +/// shows so it can be validated against golden values. +/// +public sealed class SummaryMetrics +{ + /// Total committed (allocated) bytes across all categories. + public ulong TotalAllocatedBytes { get; set; } + + /// Total resident bytes across all categories. + public ulong TotalResidentBytes { get; set; } + + /// Allocated Memory Distribution rows (Native, Managed, Executables & Mapped, Graphics, Untracked). + public List AllocatedMemoryDistribution { get; } = []; + + /// Managed Heap Utilization rows (Virtual Machine, Objects, Empty Heap Space). + public List ManagedHeapUtilization { get; } = []; +} + +/// +/// Stable identifiers and flattening for the summary_metrics table, shared by the writers and validation. +/// +public static class SummaryMetricsTable +{ + /// Group name for the Allocated Memory Distribution rows. + public const string GroupAllocatedMemoryDistribution = "AllocatedMemoryDistribution"; + + /// Group name for the Managed Heap Utilization rows. + public const string GroupManagedHeapUtilization = "ManagedHeapUtilization"; + + /// Group name for the overall totals row. + public const string GroupTotals = "Totals"; + + /// Category name for the single totals row (committed = Total Allocated, resident = Total Resident). + public const string CategoryTotal = "Total"; + + /// Flattens summary metrics into one row per category for storage in summary_metrics. + public static IEnumerable<(string Group, string Category, ulong Committed, ulong Resident, bool ResidentAvailable)> Enumerate(SummaryMetrics metrics) + { + yield return (GroupTotals, CategoryTotal, metrics.TotalAllocatedBytes, metrics.TotalResidentBytes, true); + + foreach (var row in metrics.AllocatedMemoryDistribution) + yield return (GroupAllocatedMemoryDistribution, row.Name, row.CommittedBytes, row.ResidentBytes, row.ResidentAvailable); + + foreach (var row in metrics.ManagedHeapUtilization) + yield return (GroupManagedHeapUtilization, row.Name, row.CommittedBytes, row.ResidentBytes, row.ResidentAvailable); + } +} + +/// +/// One summary breakdown row: a named category with committed and resident byte totals. +/// +public sealed class SummaryCategory +{ + /// Category label, matching the Memory Profiler summary row (e.g. "Native", "Graphics (Estimated)"). + public string Name { get; set; } = string.Empty; + + /// Committed (allocated) bytes for this category. + public ulong CommittedBytes { get; set; } + + /// Resident bytes for this category (meaningful only when is true). + public ulong ResidentBytes { get; set; } + + /// + /// False for categories where resident size cannot be measured (Graphics, Untracked); those compare committed only. + /// + public bool ResidentAvailable { get; set; } = true; +} diff --git a/Core/Parser/ManagedSnapshotCrawler.cs b/Core/Parser/ManagedSnapshotCrawler.cs index b8501e9..d2a92a8 100644 --- a/Core/Parser/ManagedSnapshotCrawler.cs +++ b/Core/Parser/ManagedSnapshotCrawler.cs @@ -74,6 +74,8 @@ private ManagedCrawlResult CrawlInternal() TryEnsureManagedObject(address, $"gc-handle[{gcHandleIndex}]"); } + SeedStaticFieldRoots(); + while (_crawlQueue.Count > 0) { var address = _crawlQueue.Dequeue(); @@ -340,6 +342,96 @@ private IEnumerable EnumerateValueTypeReferences(ulong valueBaseAddress, } } + /// + /// Seeds the crawl from static field roots: each managed type's static field bytes are scanned for + /// references the same way Memory Profiler's static-fields crawler does, so objects reachable only via + /// statics (not from a GC handle) are discovered. Without this, that memory is misreported as empty heap. + /// + private void SeedStaticFieldRoots() + { + var staticBytes = _snapshot.ManagedTypeStaticFieldBytes; + if (staticBytes.Length == 0) + return; + + var typeCount = _snapshot.ManagedTypeNames.Length; + for (var typeIndex = 0; typeIndex < typeCount && typeIndex < staticBytes.Length; typeIndex++) + { + var blob = staticBytes[typeIndex]; + if (blob is null || blob.Length == 0) + continue; + + foreach (var fieldIndex in _snapshot.ManagedTypeFieldIndices[typeIndex]) + { + if ((uint)fieldIndex >= (uint)_snapshot.FieldIsStatic.Length) + continue; + if (_snapshot.FieldIsStatic[fieldIndex] == 0) + continue; + + var offset = _snapshot.FieldOffsets[fieldIndex]; + if (offset < 0) + continue; + + var fieldTypeIndex = _snapshot.FieldTypeIndices[fieldIndex]; + if (fieldTypeIndex < 0 || fieldTypeIndex >= typeCount) + continue; + + CrawlStaticBlobReference(blob, offset, fieldTypeIndex, recursionDepth: 0); + } + } + } + + private void CrawlStaticBlobReference(byte[] blob, long offset, int fieldTypeIndex, int recursionDepth) + { + if (recursionDepth > 24) + return; + + if (!IsValueType(fieldTypeIndex)) + { + if (TryReadPointerFromBlob(blob, offset, out var target) && target != 0) + TryEnsureManagedObject(target, "static field"); + return; + } + + // Value-type static field: walk its reference subfields in place within the static blob. + foreach (var subFieldIndex in GetInstanceFieldIndices(fieldTypeIndex)) + { + if (_snapshot.FieldIsStatic[subFieldIndex] != 0) + continue; + + var adjustedOffset = _snapshot.FieldOffsets[subFieldIndex] - (int)_vm.ObjectHeaderSize; + if (adjustedOffset < 0) + continue; + + var subTypeIndex = _snapshot.FieldTypeIndices[subFieldIndex]; + if (subTypeIndex < 0 || subTypeIndex >= _snapshot.ManagedTypeNames.Length) + continue; + + if (IsValueType(subTypeIndex)) + { + if (subTypeIndex == fieldTypeIndex) + continue; + CrawlStaticBlobReference(blob, offset + adjustedOffset, subTypeIndex, recursionDepth + 1); + } + else if (TryReadPointerFromBlob(blob, offset + adjustedOffset, out var target) && target != 0) + { + TryEnsureManagedObject(target, "static value-type field"); + } + } + } + + private bool TryReadPointerFromBlob(byte[] blob, long offset, out ulong value) + { + value = 0; + if (offset < 0 || checked(offset + _vm.PointerSize) > blob.Length) + return false; + + var span = blob.AsSpan(checked((int)offset)); + value = _vm.PointerSize == 8 + ? BinaryPrimitives.ReadUInt64LittleEndian(span) + : BinaryPrimitives.ReadUInt32LittleEndian(span); + return true; + } + private int[] GetInstanceFieldIndices(int typeIndex) { EnsureValidTypeIndex(typeIndex, "enumerate fields"); diff --git a/Core/Parser/MemoryMapResidentAggregator.cs b/Core/Parser/MemoryMapResidentAggregator.cs new file mode 100644 index 0000000..f4befd7 --- /dev/null +++ b/Core/Parser/MemoryMapResidentAggregator.cs @@ -0,0 +1,158 @@ +using System.Collections; + +namespace MemorySnapshotDataTools.Parser; + +/// +/// Assigns resident bytes to native roots and objects by walking sorted address points, +/// mirroring EntriesMemoryMapCache.ForEachFlatWithResidentSize in the Unity Memory Profiler. +/// +internal static class MemoryMapResidentAggregator +{ + private enum PointKind + { + End = 0, + Start = 1, + SystemMemoryRegion = 2, + } + + private readonly record struct MemoryPoint(ulong Address, PointKind Kind, int Index, bool IsAllocation); + + /// + /// Computes per-root and per-object resident sizes from page bitmap data and address layout. + /// + public static (ulong[] RootResidentSizes, ulong[] ObjectResidentSizes) Compute(DecodedSnapshot decoded) + { + var rootCount = decoded.NativeRootIds.Length; + var objectCount = decoded.NativeObjectNames.Length; + var rootResident = new ulong[rootCount]; + var objectResident = new ulong[objectCount]; + + if (!ResidentMemoryCalculator.HasPerObjectResident(decoded)) + return (rootResident, objectResident); + + var rootIdToIndex = BuildRootIdToIndex(decoded); + var points = BuildSortedPoints(decoded); + if (points.Count < 2) + return (rootResident, objectResident); + + var pageStates = new BitArray(decoded.SystemMemoryResidentPageStates[0]); + var pageSize = decoded.SystemMemoryResidentPageSize; + var currentRegionIndex = -1; + + for (var i = 0; i < points.Count - 1; i++) + { + var cur = points[i]; + var next = points[i + 1]; + + if (cur.Kind == PointKind.SystemMemoryRegion) + currentRegionIndex = cur.Index; + + if (cur.Kind != PointKind.Start || currentRegionIndex < 0) + continue; + + var size = next.Address - cur.Address; + if (size == 0) + continue; + + var resident = ResidentMemoryCalculator.CalculateResidentForRange( + decoded, + pageStates, + pageSize, + currentRegionIndex, + cur.Address, + size); + + if (cur.IsAllocation) + { + if (cur.Index >= decoded.NativeAllocationRootReferenceIds.Length) + continue; + + var rootReferenceId = decoded.NativeAllocationRootReferenceIds[cur.Index]; + if (rootReferenceId < 1) + continue; + + if (rootIdToIndex.TryGetValue(rootReferenceId, out var allocationRootIndex)) + rootResident[allocationRootIndex] += resident; + } + else if (cur.Kind == PointKind.Start) + { + objectResident[cur.Index] += resident; + if (cur.Index < decoded.NativeObjectRootReferenceIds.Length) + { + var rootReferenceId = decoded.NativeObjectRootReferenceIds[cur.Index]; + if (rootReferenceId >= 1 && + rootIdToIndex.TryGetValue(rootReferenceId, out var objectRootIndex)) + { + rootResident[objectRootIndex] += resident; + } + } + } + } + + return (rootResident, objectResident); + } + + private static Dictionary BuildRootIdToIndex(DecodedSnapshot decoded) + { + var map = new Dictionary(decoded.NativeRootIds.Length); + for (var i = 0; i < decoded.NativeRootIds.Length; i++) + map[decoded.NativeRootIds[i]] = i; + return map; + } + + private static List BuildSortedPoints(DecodedSnapshot decoded) + { + var points = new List(); + + for (var i = 0; i < decoded.SystemMemoryRegionAddresses.Length; i++) + points.Add(new MemoryPoint(decoded.SystemMemoryRegionAddresses[i], PointKind.SystemMemoryRegion, i, false)); + + for (var i = 0; i < decoded.NativeAllocationAddresses.Length; i++) + { + var address = decoded.NativeAllocationAddresses[i]; + if (address == 0) + continue; + + var size = i < decoded.NativeAllocationSizes.Length ? decoded.NativeAllocationSizes[i] : 0; + if (size == 0) + continue; + + points.Add(new MemoryPoint(address, PointKind.Start, i, true)); + points.Add(new MemoryPoint(address + size, PointKind.End, i, true)); + } + + for (var i = 0; i < decoded.NativeObjectAddresses.Length; i++) + { + var address = decoded.NativeObjectAddresses[i]; + if (address == 0) + continue; + + var size = i < decoded.NativeObjectSizes.Length ? decoded.NativeObjectSizes[i] : 0; + if (size == 0) + continue; + + points.Add(new MemoryPoint(address, PointKind.Start, i, false)); + points.Add(new MemoryPoint(address + size, PointKind.End, i, false)); + } + + points.Sort(static (a, b) => + { + var cmp = a.Address.CompareTo(b.Address); + if (cmp != 0) + return cmp; + + return SortOrder(a).CompareTo(SortOrder(b)); + }); + + return points; + } + + private static int SortOrder(MemoryPoint point) => point.Kind switch + { + PointKind.End => 0, + PointKind.SystemMemoryRegion => 1, + PointKind.Start when point.IsAllocation => 2, + PointKind.Start => 3, + _ => 4, + }; +} diff --git a/Core/Parser/ResidentMemoryCalculator.cs b/Core/Parser/ResidentMemoryCalculator.cs new file mode 100644 index 0000000..d09565e --- /dev/null +++ b/Core/Parser/ResidentMemoryCalculator.cs @@ -0,0 +1,212 @@ +using System.Collections; + +namespace MemorySnapshotDataTools.Parser; + +/// +/// Computes per-native-object and per-allocation resident memory sizes by intersecting +/// address ranges with OS page residency bitmaps from SystemMemoryResidentPages_* entries. +/// Only produces non-zero values for format v17+ snapshots with resident page data. +/// +internal static class ResidentMemoryCalculator +{ + /// + /// Returns whether per-object resident sizes can be computed from page bitmap data. + /// + public static bool HasPerObjectResident(DecodedSnapshot decoded) => + decoded.FormatVersion >= SnapFormatVersion.SystemMemoryResidentPagesVersion && + decoded.SystemMemoryResidentPageAddresses.Length > 0 && + decoded.SystemMemoryResidentPageSize > 0 && + decoded.SystemMemoryResidentPageStates.Length > 0 && + decoded.SystemMemoryResidentPageStates[0].Length > 0; + + /// + /// Computes resident_size_bytes for each native object address range. + /// + public static ulong[] ComputePerObject(DecodedSnapshot decoded) + { + var count = decoded.NativeObjectNames.Length; + var result = new ulong[count]; + if (!HasPerObjectResident(decoded)) + return result; + + var pageStates = new BitArray(decoded.SystemMemoryResidentPageStates[0]); + var pageSize = decoded.SystemMemoryResidentPageSize; + + for (var i = 0; i < count; i++) + { + if (i >= decoded.NativeObjectAddresses.Length || i >= decoded.NativeObjectSizes.Length) + continue; + + var address = decoded.NativeObjectAddresses[i]; + var size = decoded.NativeObjectSizes[i]; + if (size == 0) + continue; + + var regionIndex = FindResidentPageRegionIndex(decoded, address); + if (regionIndex < 0) + continue; + + result[i] = CalculateResidentForRange( + decoded, + pageStates, + pageSize, + regionIndex, + address, + size); + } + + return result; + } + + /// + /// Computes resident bytes for each native allocation. + /// + public static ulong[] ComputePerAllocation(DecodedSnapshot decoded) + { + var count = decoded.NativeAllocationAddresses.Length; + var result = new ulong[count]; + if (!HasPerObjectResident(decoded)) + return result; + + var pageStates = new BitArray(decoded.SystemMemoryResidentPageStates[0]); + var pageSize = decoded.SystemMemoryResidentPageSize; + + for (var i = 0; i < count; i++) + { + var address = decoded.NativeAllocationAddresses[i]; + var size = decoded.NativeAllocationSizes[i]; + if (size == 0) + continue; + + var regionIndex = FindResidentPageRegionIndex(decoded, address); + if (regionIndex < 0) + continue; + + result[i] = CalculateResidentForRange( + decoded, + pageStates, + pageSize, + regionIndex, + address, + size); + } + + return result; + } + + /// + /// Aggregates allocation resident bytes onto native roots by RootReferenceId, + /// and adds native object resident bytes for objects linked to the same root. + /// + public static ulong[] ComputePerRoot( + DecodedSnapshot decoded, + ulong[] objectResidentSizes, + ulong[] allocationResidentSizes) + { + var rootCount = decoded.NativeRootIds.Length; + var result = new ulong[rootCount]; + if (!HasPerObjectResident(decoded)) + return result; + + var rootIdToIndex = new Dictionary(rootCount); + for (var i = 0; i < rootCount; i++) + rootIdToIndex[decoded.NativeRootIds[i]] = i; + + for (var i = 0; i < allocationResidentSizes.Length; i++) + { + if (i >= decoded.NativeAllocationRootReferenceIds.Length) + continue; + + var rootId = decoded.NativeAllocationRootReferenceIds[i]; + if (rootId < 0 || !rootIdToIndex.TryGetValue(rootId, out var rootIndex)) + continue; + + result[rootIndex] += allocationResidentSizes[i]; + } + + for (var i = 0; i < objectResidentSizes.Length; i++) + { + if (i >= decoded.NativeObjectRootReferenceIds.Length) + continue; + + var rootId = decoded.NativeObjectRootReferenceIds[i]; + if (rootId < 0 || !rootIdToIndex.TryGetValue(rootId, out var rootIndex)) + continue; + + result[rootIndex] += objectResidentSizes[i]; + } + + return result; + } + + private static int FindResidentPageRegionIndex(DecodedSnapshot decoded, ulong address) + { + var regions = decoded.SystemMemoryResidentPageAddresses; + var best = -1; + for (var i = 0; i < regions.Length; i++) + { + if (address < regions[i]) + break; + + var regionEnd = i + 1 < regions.Length + ? regions[i + 1] + : ulong.MaxValue; + + if (address < regionEnd || i == regions.Length - 1) + { + best = i; + break; + } + + best = i; + } + + return best; + } + + internal static ulong CalculateResidentForRange( + DecodedSnapshot decoded, + BitArray pageStates, + ulong pageSizeUlong, + int regionIndex, + ulong address, + ulong size) + { + if (size == 0 || pageSizeUlong == 0) + return 0; + + var pageSize = pageSizeUlong; + var regionAddress = decoded.SystemMemoryResidentPageAddresses[regionIndex]; + var firstPageIndex = decoded.SystemMemoryResidentPageFirstIndices[regionIndex]; + var lastPageIndex = decoded.SystemMemoryResidentPageLastIndices[regionIndex]; + + var addrDelta = address - regionAddress; + var begPage = (int)(addrDelta / pageSize) + firstPageIndex; + var endPage = (int)((addrDelta + size - 1) / pageSize) + firstPageIndex; + + if (begPage < firstPageIndex || endPage > lastPageIndex) + return 0; + + ulong residentSize = 0; + for (var p = begPage; p <= endPage; p++) + { + if (p >= 0 && p < pageStates.Length && pageStates[p]) + residentSize += pageSize; + } + + if (begPage >= 0 && begPage < pageStates.Length && pageStates[begPage]) + { + var head = address % pageSize; + residentSize -= head; + } + + if (endPage >= 0 && endPage < pageStates.Length && pageStates[endPage]) + { + var tail = (address + size) % pageSize; + if (tail > 0) + residentSize -= pageSize - tail; + } + + return residentSize; + } +} diff --git a/Core/Parser/SnapDataModel.cs b/Core/Parser/SnapDataModel.cs index 3984c62..dba0b31 100644 --- a/Core/Parser/SnapDataModel.cs +++ b/Core/Parser/SnapDataModel.cs @@ -21,6 +21,8 @@ internal enum SnapEntryType : ushort { Metadata_Version = 0, Metadata_RecordDate = 1, + Metadata_UserMetadata = 2, + Metadata_CaptureFlags = 3, Metadata_VirtualMachineInformation = 4, NativeTypes_Name = 5, NativeTypes_NativeBaseTypeArrayIndex = 6, @@ -29,7 +31,9 @@ internal enum SnapEntryType : ushort NativeObjects_Flags = 9, NativeObjects_InstanceId = 10, NativeObjects_Name = 11, + NativeObjects_NativeObjectAddress = 12, NativeObjects_Size = 13, + NativeObjects_RootReferenceId = 14, GCHandles_Target = 15, Connections_From = 16, Connections_To = 17, @@ -39,6 +43,7 @@ internal enum SnapEntryType : ushort TypeDescriptions_Name = 23, TypeDescriptions_Assembly = 24, TypeDescriptions_FieldIndices = 25, + TypeDescriptions_StaticFieldBytes = 26, TypeDescriptions_BaseOrElementTypeIndex = 27, TypeDescriptions_Size = 28, TypeDescriptions_TypeInfoAddress = 29, @@ -51,6 +56,7 @@ internal enum SnapEntryType : ushort NativeRootReferences_ObjectName = 37, NativeRootReferences_AccumulatedSize = 38, NativeAllocations_MemoryRegionIndex = 39, + NativeAllocations_RootReferenceId = 40, NativeAllocations_Address = 42, NativeAllocations_Size = 43, NativeAllocations_OverheadSize = 44, @@ -64,6 +70,18 @@ internal enum SnapEntryType : ushort NativeMemoryLabels_Name = 52, NativeObjects_GCHandleIndex = 58, NativeObjects_GCHandleIndex_Legacy = 62, + ProfileTarget_Info = 59, + ProfileTarget_MemoryStats = 60, + SystemMemoryRegions_Address = 83, + SystemMemoryRegions_Size = 84, + SystemMemoryRegions_Resident = 85, + SystemMemoryRegions_Type = 86, + SystemMemoryRegions_Name = 87, + SystemMemoryResidentPages_Address = 88, + SystemMemoryResidentPages_FirstPageIndex = 89, + SystemMemoryResidentPages_LastPageIndex = 90, + SystemMemoryResidentPages_PagesState = 91, + SystemMemoryResidentPages_PageSize = 92, } /// Format version constants used when decoding snapshot entries (e.g. instance IDs, heap sections). @@ -77,6 +95,38 @@ internal static class SnapFormatVersion /// Version for memory label size and heap ID in heap section metadata. public const uint MemLabelSizeAndHeapIdVersion = 12; + + /// Version at which OS system memory regions are present (entries 83–87). + public const uint SystemMemoryRegionsVersion = 16; + + /// Version at which per-page resident bitmaps are present (entries 88–92). + public const uint SystemMemoryResidentPagesVersion = 17; +} + +/// +/// Type of a managed heap section, decoded from the high bit of its start address. +/// Mirrors Unity Memory Profiler's ManagedMemorySectionEntriesCache.MemorySectionType. +/// +public enum ManagedHeapSectionKind : byte +{ + /// Garbage collector heap section (reported as "Empty Heap Space" in the summary). + GarbageCollector = 0, + + /// Virtual machine heap section (reported as "Virtual Machine" in the summary). + VirtualMachine = 1, +} + +/// +/// Process-wide memory statistics reported by the profiling target (ProfileTarget_MemoryStats). +/// Used by the summary builder for graphics estimation and the legacy untracked fallback. +/// +public sealed class DecodedTargetMemoryStats +{ + /// Total virtual memory committed by the process, as reported by the OS. + public ulong TotalVirtualMemory { get; set; } + + /// Estimated graphics memory used, as reported by the graphics driver. + public ulong GraphicsUsedMemory { get; set; } } /// @@ -116,6 +166,9 @@ public sealed class DecodedSnapshot /// Record date in .NET ticks (UTC). public long RecordDateTicksUtc { get; set; } + /// Session, platform, and Unity version from snapshot metadata entries. + public CaptureMetadata CaptureMetadata { get; set; } = new(); + /// Native type display names. public string[] NativeTypeNames { get; set; } = []; @@ -131,6 +184,12 @@ public sealed class DecodedSnapshot /// Per-native-object size in bytes. public ulong[] NativeObjectSizes { get; set; } = []; + /// Per-native-object base address, or 0 when not present in the capture. + public ulong[] NativeObjectAddresses { get; set; } = []; + + /// Per-native-object root reference ID linking to , or -1. + public long[] NativeObjectRootReferenceIds { get; set; } = []; + /// Per-native-object flags (e.g. destroyed). public int[] NativeObjectFlags { get; set; } = []; @@ -194,15 +253,54 @@ public sealed class DecodedSnapshot /// Memory region index per allocation, or -1. public int[] NativeAllocationMemoryRegionIndices { get; set; } = []; + /// Root reference ID per allocation, or -1. + public long[] NativeAllocationRootReferenceIds { get; set; } = []; + + /// OS system memory region base addresses (format v16+). + public ulong[] SystemMemoryRegionAddresses { get; set; } = []; + + /// OS system memory region committed sizes in bytes (format v16+). + public ulong[] SystemMemoryRegionSizes { get; set; } = []; + + /// OS system memory region resident sizes in bytes (format v16+). + public ulong[] SystemMemoryRegionResidentSizes { get; set; } = []; + + /// OS system memory region type codes (format v16+). + public int[] SystemMemoryRegionTypes { get; set; } = []; + + /// OS system memory region names (format v16+). + public string[] SystemMemoryRegionNames { get; set; } = []; + + /// Resident-page range base addresses, one per system region (format v17+). + public ulong[] SystemMemoryResidentPageAddresses { get; set; } = []; + + /// First page index in the global page bitmap per system region (format v17+). + public int[] SystemMemoryResidentPageFirstIndices { get; set; } = []; + + /// Last page index in the global page bitmap per system region (format v17+). + public int[] SystemMemoryResidentPageLastIndices { get; set; } = []; + + /// Global page residency bitmap bytes (format v17+); one element holds the full bitset. + public byte[][] SystemMemoryResidentPageStates { get; set; } = []; + + /// Page size in bytes for resident page calculations (format v17+). + public ulong SystemMemoryResidentPageSize { get; set; } + /// VM layout (pointer size, header offsets). public DecodedVirtualMachineInfo VirtualMachineInformation { get; set; } = new(); - /// Start address of each managed heap section. + /// Start address of each managed heap section (high type-bit masked off). public ulong[] ManagedHeapSectionStartAddresses { get; set; } = []; + /// Type (VM vs GC) of each managed heap section, parallel to . + public ManagedHeapSectionKind[] ManagedHeapSectionTypes { get; set; } = []; + /// Raw bytes of each managed heap section. public byte[][] ManagedHeapSectionBytes { get; set; } = []; + /// Process memory statistics from the capture target, or null when absent. + public DecodedTargetMemoryStats? TargetMemoryStats { get; set; } + /// Managed type flags (value type, array, etc.). public int[] ManagedTypeFlags { get; set; } = []; @@ -224,6 +322,9 @@ public sealed class DecodedSnapshot /// Per-type array of field description indices. public int[][] ManagedTypeFieldIndices { get; set; } = []; + /// Per-type static field byte blob (static field values), parallel to ; empty when absent. + public byte[][] ManagedTypeStaticFieldBytes { get; set; } = []; + /// Field offset in bytes. public int[] FieldOffsets { get; set; } = []; diff --git a/Core/Parser/SnapMetadataReader.cs b/Core/Parser/SnapMetadataReader.cs new file mode 100644 index 0000000..3ab9eb6 --- /dev/null +++ b/Core/Parser/SnapMetadataReader.cs @@ -0,0 +1,117 @@ +using MemorySnapshotDataTools; + +namespace MemorySnapshotDataTools.Parser; + +/// +/// Reads capture metadata (session, platform, Unity version) from a .snap file without full decode. +/// +public static class SnapMetadataReader +{ + /// + /// Reads metadata from the snapshot at . + /// + public static CaptureMetadata Read(string snapPath) + { + using var reader = SnapReader.Open(snapPath); + return Read(reader, snapPath); + } + + /// + /// Reads metadata from an open . + /// + internal static CaptureMetadata Read(SnapReader reader, string? snapPath = null) + { + var formatVersion = reader.ReadMetadataVersion(); + var ticks = reader.ReadMetadataRecordDateTicks(); + DateTime? recordDate = ticks > 0 ? new DateTime(ticks, DateTimeKind.Utc) : null; + + var metadata = SnapProfileTargetInfoParser.TryRead(reader, formatVersion) + ?? TryReadLegacyPlatform(reader) + ?? new CaptureMetadata(); + + metadata = metadata with + { + SnapFormatVersion = formatVersion, + RecordDateUtc = recordDate, + }; + + if (string.IsNullOrWhiteSpace(metadata.Platform) && !string.IsNullOrWhiteSpace(snapPath)) + metadata = metadata with { Platform = InferPlatformFromFileName(snapPath) }; + + return metadata; + } + + /// + /// Infers runtime platform from snapshot filename tokens (IOS, Android, etc.). + /// + public static string InferPlatformFromFileName(string path) + { + var name = Path.GetFileNameWithoutExtension(path); + if (name.Contains("IOS", StringComparison.OrdinalIgnoreCase) + || name.Contains("IPhone", StringComparison.OrdinalIgnoreCase)) + { + return "IPhonePlayer"; + } + + if (name.Contains("Android", StringComparison.OrdinalIgnoreCase)) + return "Android"; + + return string.Empty; + } + + private static CaptureMetadata? TryReadLegacyPlatform(SnapReader reader) + { + if (!reader.HasEntry(SnapEntryType.Metadata_UserMetadata)) + return null; + + var blob = reader.ReadSingleElementBytes(SnapEntryType.Metadata_UserMetadata); + if (blob.Length < 8) + return null; + + try + { + return TryParseLegacyUserMetadata(blob); + } + catch + { + return null; + } + } + + /// + /// Legacy user metadata: int32 description length, UTF-16 description, int32 platform length, UTF-16 platform. + /// + private static CaptureMetadata? TryParseLegacyUserMetadata(byte[] buffer) + { + var offset = 0; + offset = SkipUtf16String(buffer, offset, out _); + if (offset < 0 || offset + 4 > buffer.Length) + return null; + + var platformLen = BitConverter.ToInt32(buffer, offset); + offset += 4; + if (platformLen <= 0 || offset + platformLen * 2 > buffer.Length) + return new CaptureMetadata { Platform = "Unknown Platform" }; + + var platform = System.Text.Encoding.Unicode.GetString(buffer, offset, platformLen * 2); + return new CaptureMetadata { Platform = platform }; + } + + private static int SkipUtf16String(byte[] buffer, int offset, out string value) + { + value = string.Empty; + if (offset + 4 > buffer.Length) + return -1; + + var len = BitConverter.ToInt32(buffer, offset); + offset += 4; + if (len == 0) + return offset; + + if (offset + len * 2 > buffer.Length) + return -1; + + value = System.Text.Encoding.Unicode.GetString(buffer, offset, len * 2); + return offset + len * 2; + } +} diff --git a/Core/Parser/SnapProfileTargetInfoParser.cs b/Core/Parser/SnapProfileTargetInfoParser.cs new file mode 100644 index 0000000..5363b68 --- /dev/null +++ b/Core/Parser/SnapProfileTargetInfoParser.cs @@ -0,0 +1,143 @@ +using System.Buffers.Binary; +using System.Text; +using MemorySnapshotDataTools; + +namespace MemorySnapshotDataTools.Parser; + +/// +/// Parses the 512-byte ProfileTarget_Info blob from Unity memory snapshots. +/// Layout matches Unity Memory Profiler ProfileTargetInfo. +/// +internal static class SnapProfileTargetInfoParser +{ + private const int StructSize = 512; + private const int OffSessionGuid = 0; + private const int OffRuntimePlatform = 4; + private const int OffUnityVersionLength = 48; + private const int OffUnityVersionBuffer = 52; + private const int OffProductNameLength = 68; + private const int OffProductNameBuffer = 72; + + /// Minimum snap format that may include profile target info. + public const uint MinFormatWithProfileTarget = 11; + + /// + /// Parses profile target bytes when at least bytes are available. + /// + public static bool TryParse(ReadOnlySpan data, out CaptureMetadata metadata) + { + metadata = new CaptureMetadata(); + if (data.Length < OffProductNameBuffer) + return false; + + var sessionGuid = BinaryPrimitives.ReadUInt32LittleEndian(data[OffSessionGuid..]); + var runtimePlatform = BinaryPrimitives.ReadInt32LittleEndian(data[OffRuntimePlatform..]); + var unityLen = BinaryPrimitives.ReadUInt32LittleEndian(data[OffUnityVersionLength..]); + var productLen = BinaryPrimitives.ReadUInt32LittleEndian(data[OffProductNameLength..]); + + if (unityLen == 0 || unityLen > 16 || productLen == 0 || productLen > 256) + return false; + + var unityVersion = ReadUtf8String(data, OffUnityVersionBuffer, 16, unityLen); + var productName = ReadUtf8String(data, OffProductNameBuffer, 256, productLen); + if (string.IsNullOrWhiteSpace(unityVersion) || string.IsNullOrWhiteSpace(productName)) + return false; + + var platform = RuntimePlatformToName(runtimePlatform); + if (platform.StartsWith("Platform_", StringComparison.Ordinal) && sessionGuid == 0) + return false; + + metadata = new CaptureMetadata + { + SessionGuid = sessionGuid, + ProductName = productName, + UnityVersion = unityVersion, + Platform = platform, + }; + return true; + } + + /// Reads profile target info from a snapshot reader when present. + public static CaptureMetadata? TryRead(SnapReader reader, uint formatVersion) + { + if (formatVersion < MinFormatWithProfileTarget) + return null; + + CaptureMetadata? best = null; + + if (reader.HasEntry(SnapEntryType.ProfileTarget_Info)) + { + best = TryReadEntryBlob(reader, SnapEntryType.ProfileTarget_Info); + } + + if (best is { HasProfilerSession: true }) + return best; + + for (ushort i = 0; i < 128; i++) + { + var entryType = (SnapEntryType)i; + if (entryType == SnapEntryType.ProfileTarget_Info) + continue; + if (!reader.HasEntry(entryType)) + continue; + + var candidate = TryReadEntryBlob(reader, entryType); + if (candidate is null) + continue; + + if (candidate.HasProfilerSession) + return candidate; + + best ??= candidate; + } + + return best; + } + + private static CaptureMetadata? TryReadEntryBlob(SnapReader reader, SnapEntryType entryType) + { + try + { + var bytes = reader.ReadSingleElementBytes(entryType); + if (bytes.Length < StructSize) + return null; + + for (var start = 0; start <= bytes.Length - StructSize; start += 4) + { + if (TryParse(bytes.AsSpan(start, StructSize), out var meta)) + return meta; + } + } + catch (InvalidOperationException) + { + return null; + } + + return null; + } + + private static string ReadUtf8String(ReadOnlySpan data, int offset, int maxLen, uint length) + { + if (length == 0 || length > maxLen) + return string.Empty; + if (offset + length > data.Length) + return string.Empty; + return Encoding.UTF8.GetString(data.Slice(offset, (int)length)); + } + + private static string RuntimePlatformToName(int runtimePlatform) => runtimePlatform switch + { + 8 => "IPhonePlayer", + 11 => "Android", + 31 => "tvOS", + 9 => "PS4", + 38 => "PS5", + 2 => "WindowsPlayer", + 1 => "OSXPlayer", + 13 => "LinuxPlayer", + 0 => "OSXEditor", + 7 => "WindowsEditor", + 12 => "LinuxEditor", + _ => $"Platform_{runtimePlatform}", + }; +} diff --git a/Core/Parser/SnapReader.cs b/Core/Parser/SnapReader.cs index 3bf006e..c2ac7a2 100644 --- a/Core/Parser/SnapReader.cs +++ b/Core/Parser/SnapReader.cs @@ -269,6 +269,58 @@ private T ReadSingle(SnapEntryType entryType) where T : unmanaged return arr[0]; } + /// + /// Reads the raw byte blob for a single-element entry, or the first element of a constant-size array. + /// + internal byte[] ReadSingleElementBytes(SnapEntryType entryType) + { + EnsureDefined(entryType); + var entry = _entries[(int)entryType]; + if (entry.Format == SnapEntryFormat.SingleElement) + return ReadConstEntryBytes(entry, 0, 1); + + if (entry.Format == SnapEntryFormat.ConstantSizeElementArray && entry.Count > 0) + return ReadConstEntryBytes(entry, 0, 1); + + if (entry.Format == SnapEntryFormat.DynamicSizeElementArray && entry.Count > 0) + { + GetDynamicElementBounds(entry, 0, out var start, out var length); + return ReadBlockRange(_blocks[checked((int)entry.BlockIndex)], start, checked((int)length)); + } + + throw new InvalidOperationException($"Entry '{entryType}' cannot be read as a single blob."); + } + + /// + /// Reads raw bytes for a contiguous range of constant-size elements (or the single-element blob). + /// Used for resident page bitmaps stored as one constant-size blob per entry. + /// + internal byte[] ReadConstantRangeBytes(SnapEntryType entryType, int startIndex, int count) + { + EnsureDefined(entryType); + var entry = _entries[(int)entryType]; + return ReadConstEntryBytes(entry, startIndex, count); + } + + /// + /// Reads up to leading bytes of an entry's data block, starting at the + /// first element. Used for fixed-layout struct blobs (e.g. ProfileTarget_MemoryStats) whose stored + /// element size/count does not describe the full struct, mirroring Unity's ReadUnsafe(..., sizeof(T), 0, 1). + /// + internal byte[] ReadEntryLeadingBytes(SnapEntryType entryType, int byteCount) + { + EnsureDefined(entryType); + var entry = _entries[(int)entryType]; + if (entry.Format == SnapEntryFormat.DynamicSizeElementArray) + throw new InvalidOperationException($"Entry '{entryType}' is dynamic; use a dynamic read."); + + var start = entry.Format == SnapEntryFormat.SingleElement ? checked((long)entry.HeaderMeta) : 0L; + var block = _blocks[checked((int)entry.BlockIndex)]; + var available = checked((long)block.TotalBytes) - start; + var length = (int)Math.Max(0, Math.Min(byteCount, available)); + return ReadBlockRange(block, start, length); + } + private byte[] ReadConstEntryBytes(EntryData entry, int startIndex, int count) { if (entry.Format == SnapEntryFormat.SingleElement && startIndex == 0 && count == 1) diff --git a/Core/Parser/SnapSectionDecoders.cs b/Core/Parser/SnapSectionDecoders.cs index 86afbd8..38513dc 100644 --- a/Core/Parser/SnapSectionDecoders.cs +++ b/Core/Parser/SnapSectionDecoders.cs @@ -18,6 +18,7 @@ internal static class SnapSectionDecoders public static DecodedSnapshot DecodeAll(SnapReader reader) { var formatVersion = reader.ReadMetadataVersion(); + var captureMetadata = SnapMetadataReader.Read(reader); var nativeObjectTypeIndices = ReadInts(reader, SnapEntryType.NativeObjects_NativeTypeArrayIndex); var nativeObjectCount = nativeObjectTypeIndices.Length; var nativeObjectInstanceIds = ReadInstanceIds(reader, formatVersion); @@ -38,9 +39,12 @@ public static DecodedSnapshot DecodeAll(SnapReader reader) { FormatVersion = formatVersion, RecordDateTicksUtc = reader.ReadMetadataRecordDateTicks(), + CaptureMetadata = captureMetadata, NativeObjectTypeIndices = nativeObjectTypeIndices, NativeObjectInstanceIds = nativeObjectInstanceIds, NativeObjectSizes = ReadULongs(reader, SnapEntryType.NativeObjects_Size), + NativeObjectAddresses = ReadULongsWithCount(reader, SnapEntryType.NativeObjects_NativeObjectAddress, nativeObjectCount), + NativeObjectRootReferenceIds = ReadLongsWithCount(reader, SnapEntryType.NativeObjects_RootReferenceId, nativeObjectCount, -1), NativeObjectFlags = ReadIntsWithCount(reader, SnapEntryType.NativeObjects_Flags, nativeObjectCount, 0), NativeObjectGcHandleIndices = nativeObjectGcHandleIndices, GcHandleTargets = gcHandleTargets, @@ -58,13 +62,17 @@ public static DecodedSnapshot DecodeAll(SnapReader reader) NativeAllocationOverheadSizes = ReadULongsWithCount(reader, SnapEntryType.NativeAllocations_OverheadSize, nativeAllocationCount), NativeAllocationPaddingSizes = ReadULongsWithCount(reader, SnapEntryType.NativeAllocations_PaddingSize, nativeAllocationCount), NativeAllocationMemoryRegionIndices = ReadIntsWithCount(reader, SnapEntryType.NativeAllocations_MemoryRegionIndex, nativeAllocationCount, -1), + NativeAllocationRootReferenceIds = ReadLongsWithCount(reader, SnapEntryType.NativeAllocations_RootReferenceId, nativeAllocationCount, -1), VirtualMachineInformation = ReadVirtualMachineInfo(reader), ManagedHeapSectionStartAddresses = ReadManagedHeapSectionStartAddresses(reader, formatVersion), + ManagedHeapSectionTypes = ReadManagedHeapSectionTypes(reader, formatVersion), ManagedHeapSectionBytes = ReadRequiredDynamicBytes(reader, SnapEntryType.ManagedHeapSections_Bytes), + TargetMemoryStats = ReadTargetMemoryStats(reader), ManagedTypeFlags = ReadRequiredInts(reader, SnapEntryType.TypeDescriptions_Flags), ManagedTypeNames = ReadRequiredStrings(reader, SnapEntryType.TypeDescriptions_Name), ManagedTypeAssemblies = ReadRequiredStrings(reader, SnapEntryType.TypeDescriptions_Assembly), ManagedTypeFieldIndices = ReadRequiredDynamicInts(reader, SnapEntryType.TypeDescriptions_FieldIndices), + ManagedTypeStaticFieldBytes = ReadOptionalDynamicBytes(reader, SnapEntryType.TypeDescriptions_StaticFieldBytes), ManagedTypeBaseOrElementTypeIndices = ReadRequiredInts(reader, SnapEntryType.TypeDescriptions_BaseOrElementTypeIndex), ManagedTypeSizes = ReadRequiredInts(reader, SnapEntryType.TypeDescriptions_Size), ManagedTypeInfoAddresses = ReadRequiredULongs(reader, SnapEntryType.TypeDescriptions_TypeInfoAddress), @@ -81,6 +89,26 @@ public static DecodedSnapshot DecodeAll(SnapReader reader) snapshot.NativeMemoryRegionNames = ReadStringsWithCount(reader, SnapEntryType.NativeMemoryRegions_Name, snapshot.NativeMemoryRegionAddressBases.Length); snapshot.NativeMemoryLabelNames = ReadStrings(reader, SnapEntryType.NativeMemoryLabels_Name); + if (formatVersion >= SnapFormatVersion.SystemMemoryRegionsVersion) + { + snapshot.SystemMemoryRegionAddresses = ReadOptionalULongs(reader, SnapEntryType.SystemMemoryRegions_Address); + var systemRegionCount = snapshot.SystemMemoryRegionAddresses.Length; + snapshot.SystemMemoryRegionSizes = ReadULongsWithCount(reader, SnapEntryType.SystemMemoryRegions_Size, systemRegionCount); + snapshot.SystemMemoryRegionResidentSizes = ReadULongsWithCount(reader, SnapEntryType.SystemMemoryRegions_Resident, systemRegionCount); + snapshot.SystemMemoryRegionTypes = ReadSystemMemoryRegionTypes(reader, systemRegionCount); + snapshot.SystemMemoryRegionNames = ReadStringsWithCount(reader, SnapEntryType.SystemMemoryRegions_Name, systemRegionCount); + } + + if (formatVersion >= SnapFormatVersion.SystemMemoryResidentPagesVersion) + { + snapshot.SystemMemoryResidentPageAddresses = ReadOptionalULongs(reader, SnapEntryType.SystemMemoryResidentPages_Address); + var residentPageRangeCount = snapshot.SystemMemoryResidentPageAddresses.Length; + snapshot.SystemMemoryResidentPageFirstIndices = ReadIntsWithCount(reader, SnapEntryType.SystemMemoryResidentPages_FirstPageIndex, residentPageRangeCount, 0); + snapshot.SystemMemoryResidentPageLastIndices = ReadIntsWithCount(reader, SnapEntryType.SystemMemoryResidentPages_LastPageIndex, residentPageRangeCount, 0); + snapshot.SystemMemoryResidentPageStates = ReadResidentPageStates(reader); + snapshot.SystemMemoryResidentPageSize = ReadResidentPageSize(reader); + } + ValidateLengths(snapshot); return snapshot; } @@ -96,6 +124,10 @@ private static void ValidateLengths(DecodedSnapshot snapshot) EnsureArrayLength(nativeCount, snapshot.NativeObjectGcHandleIndices.Length, "NativeObjects_GCHandleIndex"); if (snapshot.NativeObjectFlags.Length > 0) EnsureArrayLength(nativeCount, snapshot.NativeObjectFlags.Length, "NativeObjects_Flags"); + if (snapshot.NativeObjectAddresses.Length > 0) + EnsureArrayLength(nativeCount, snapshot.NativeObjectAddresses.Length, "NativeObjects_NativeObjectAddress"); + if (snapshot.NativeObjectRootReferenceIds.Length > 0) + EnsureArrayLength(nativeCount, snapshot.NativeObjectRootReferenceIds.Length, "NativeObjects_RootReferenceId"); } var rootsCount = snapshot.NativeRootIds.Length; @@ -120,8 +152,29 @@ private static void ValidateLengths(DecodedSnapshot snapshot) EnsureArrayLength(allocationCount, snapshot.NativeAllocationOverheadSizes.Length, "NativeAllocations_OverheadSize"); EnsureArrayLength(allocationCount, snapshot.NativeAllocationPaddingSizes.Length, "NativeAllocations_PaddingSize"); EnsureArrayLength(allocationCount, snapshot.NativeAllocationMemoryRegionIndices.Length, "NativeAllocations_MemoryRegionIndex"); + if (snapshot.NativeAllocationRootReferenceIds.Length > 0) + EnsureArrayLength(allocationCount, snapshot.NativeAllocationRootReferenceIds.Length, "NativeAllocations_RootReferenceId"); + + var systemRegionCount = snapshot.SystemMemoryRegionAddresses.Length; + if (systemRegionCount > 0) + { + EnsureArrayLength(systemRegionCount, snapshot.SystemMemoryRegionSizes.Length, "SystemMemoryRegions_Size"); + EnsureArrayLength(systemRegionCount, snapshot.SystemMemoryRegionResidentSizes.Length, "SystemMemoryRegions_Resident"); + if (snapshot.SystemMemoryRegionTypes.Length > 0) + EnsureArrayLength(systemRegionCount, snapshot.SystemMemoryRegionTypes.Length, "SystemMemoryRegions_Type"); + if (snapshot.SystemMemoryRegionNames.Length > 0) + EnsureArrayLength(systemRegionCount, snapshot.SystemMemoryRegionNames.Length, "SystemMemoryRegions_Name"); + } + + var residentPageCount = snapshot.SystemMemoryResidentPageAddresses.Length; + if (residentPageCount > 0) + { + EnsureArrayLength(residentPageCount, snapshot.SystemMemoryResidentPageFirstIndices.Length, "SystemMemoryResidentPages_FirstPageIndex"); + EnsureArrayLength(residentPageCount, snapshot.SystemMemoryResidentPageLastIndices.Length, "SystemMemoryResidentPages_LastPageIndex"); + } EnsureArrayLength(snapshot.ManagedHeapSectionStartAddresses.Length, snapshot.ManagedHeapSectionBytes.Length, "ManagedHeapSections_Bytes"); + EnsureArrayLength(snapshot.ManagedHeapSectionStartAddresses.Length, snapshot.ManagedHeapSectionTypes.Length, "ManagedHeapSections_Type"); var managedTypeCount = snapshot.ManagedTypeNames.Length; EnsureArrayLength(managedTypeCount, snapshot.ManagedTypeFlags.Length, "TypeDescriptions_Flags"); @@ -183,6 +236,46 @@ private static int[] ReadRequiredInts(SnapReader reader, SnapEntryType type) return reader.ReadPrimitiveArray(type); } + /// + /// Reads SystemMemoryRegions_Type as an int array. The values are serialized as ushort + /// (Unity's MemoryType : ushort); falls back to int then byte element sizes for other formats. + /// Reading with the wrong element size throws on a byte-size mismatch, which previously left every + /// region misclassified as Private/Untracked. + /// + private static int[] ReadSystemMemoryRegionTypes(SnapReader reader, int count) + { + if (!reader.HasEntry(SnapEntryType.SystemMemoryRegions_Type)) + return count > 0 ? new int[count] : []; + + try + { + var ushorts = reader.ReadPrimitiveArray(SnapEntryType.SystemMemoryRegions_Type); + if (ushorts.Length == count) + return Array.ConvertAll(ushorts, v => (int)v); + } + catch + { + // Try other element widths below. + } + + var ints = ReadOptionalInts(reader, SnapEntryType.SystemMemoryRegions_Type); + if (ints.Length == count) + return ints; + + try + { + var bytes = reader.ReadPrimitiveArray(SnapEntryType.SystemMemoryRegions_Type); + if (bytes.Length == count) + return Array.ConvertAll(bytes, v => (int)v); + } + catch + { + // Fall through to zero-filled fallback. + } + + return count > 0 ? new int[count] : []; + } + private static int[] ReadIntsWithCount(SnapReader reader, SnapEntryType type, int fallbackCount, int fallbackValue = 0) { var values = ReadOptionalInts(reader, type); @@ -270,6 +363,61 @@ private static ulong[] ReadManagedHeapSectionStartAddresses(SnapReader reader, u return unmasked; } + /// + /// Decodes each managed heap section's type from the high bit of its start address. + /// Set bit means a virtual machine section; cleared bit (or pre-v12 formats with no flag) means a GC section. + /// Mirrors Unity Memory Profiler's ManagedMemorySectionEntriesCache. + /// + private static ManagedHeapSectionKind[] ReadManagedHeapSectionTypes(SnapReader reader, uint formatVersion) + { + var starts = ReadRequiredULongs(reader, SnapEntryType.ManagedHeapSections_StartAddress); + var types = new ManagedHeapSectionKind[starts.Length]; + if (formatVersion < SnapFormatVersion.MemLabelSizeAndHeapIdVersion) + return types; + + for (var i = 0; i < starts.Length; i++) + { + types[i] = (starts[i] & HeapSectionTypeFlagMask) == HeapSectionTypeFlagMask + ? ManagedHeapSectionKind.VirtualMachine + : ManagedHeapSectionKind.GarbageCollector; + } + + return types; + } + + /// + /// Reads the ProfileTarget_MemoryStats blob (a single fixed-size struct) and extracts the + /// fields the summary builder needs. Returns null when the entry is absent or too small. + /// + private static DecodedTargetMemoryStats? ReadTargetMemoryStats(SnapReader reader) + { + if (!reader.HasEntry(SnapEntryType.ProfileTarget_MemoryStats)) + return null; + + byte[] bytes; + try + { + // The entry's stored element size/count does not describe the full struct, so read the leading + // struct bytes directly from the entry's block (mirrors Unity's ReadUnsafe(..., sizeof(struct), 0, 1)). + bytes = reader.ReadEntryLeadingBytes(SnapEntryType.ProfileTarget_MemoryStats, 40); + } + catch + { + return null; + } + + // Sequential ulong fields: TotalVirtualMemory@0, TotalUsedMemory@8, TotalReservedMemory@16, + // TempAllocatorUsedMemory@24, GraphicsUsedMemory@32. + if (bytes.Length < 40) + return null; + + return new DecodedTargetMemoryStats + { + TotalVirtualMemory = BitConverter.ToUInt64(bytes, 0), + GraphicsUsedMemory = BitConverter.ToUInt64(bytes, 32), + }; + } + private static ulong[] ReadInstanceIds(SnapReader reader, uint formatVersion) { if (!reader.HasEntry(SnapEntryType.NativeObjects_InstanceId)) @@ -397,6 +545,105 @@ private static int[] ReadOptionalInts(SnapReader reader, SnapEntryType type) private static long[] ReadLongs(SnapReader reader, SnapEntryType type) => reader.HasEntry(type) ? reader.ReadPrimitiveArray(type) : []; + private static long[] ReadLongsWithCount(SnapReader reader, SnapEntryType type, int fallbackCount, long fallbackValue = 0) + { + var values = ReadOptionalLongs(reader, type); + if (values.Length > 0) + return values; + + return fallbackCount > 0 ? Enumerable.Repeat(fallbackValue, fallbackCount).ToArray() : []; + } + + private static long[] ReadOptionalLongs(SnapReader reader, SnapEntryType type) + { + if (!reader.HasEntry(type)) + return []; + + try + { + return reader.ReadPrimitiveArray(type); + } + catch + { + return []; + } + } + + private static ulong ReadResidentPageSize(SnapReader reader) + { + if (!reader.HasEntry(SnapEntryType.SystemMemoryResidentPages_PageSize)) + return 0; + + try + { + var uints = reader.ReadPrimitiveArray(SnapEntryType.SystemMemoryResidentPages_PageSize); + if (uints.Length > 0) + return uints[0]; + } + catch + { + // Fall through to ulong read. + } + + try + { + var ulongs = reader.ReadPrimitiveArray(SnapEntryType.SystemMemoryResidentPages_PageSize); + if (ulongs.Length > 0) + return ulongs[0]; + } + catch + { + // Entry missing or unsupported format. + } + + return 0; + } + + private static byte[][] ReadResidentPageStates(SnapReader reader) + { + if (!reader.HasEntry(SnapEntryType.SystemMemoryResidentPages_PagesState)) + return []; + + try + { + var dynamic = reader.ReadDynamicByteArrays(SnapEntryType.SystemMemoryResidentPages_PagesState); + if (dynamic.Length > 0 && dynamic[0].Length > 0) + return new[] { dynamic[0] }; + } + catch + { + // Fall through to constant-size blob read. + } + + try + { + var blob = reader.ReadConstantRangeBytes(SnapEntryType.SystemMemoryResidentPages_PagesState, 0, 1); + if (blob.Length > 0) + return new[] { blob }; + } + catch + { + // Entry missing or unsupported format. + } + + return []; + } + + private static byte[][] ReadOptionalDynamicBytes(SnapReader reader, SnapEntryType type) + { + if (!reader.HasEntry(type)) + return []; + + try + { + return reader.ReadDynamicByteArrays(type); + } + catch + { + return []; + } + } + private static ulong[] ReadULongs(SnapReader reader, SnapEntryType type) => reader.HasEntry(type) ? reader.ReadPrimitiveArray(type) : []; diff --git a/Core/Parser/SnapshotBridge.cs b/Core/Parser/SnapshotBridge.cs index be50aa6..0b81a4d 100644 --- a/Core/Parser/SnapshotBridge.cs +++ b/Core/Parser/SnapshotBridge.cs @@ -37,23 +37,43 @@ public static RawSnapshotData ExtractRawData(string snapshotPath, IProgressRepor /// Validated . public static RawSnapshotData ExtractFromDecoded(DecodedSnapshot decoded, string snapshotPath) { + var captureMeta = decoded.CaptureMetadata; + var unityVersion = !string.IsNullOrWhiteSpace(captureMeta.UnityVersion) + ? captureMeta.UnityVersion + : $"format:{decoded.FormatVersion}"; + var data = new RawSnapshotData { SnapshotInfo = new SnapshotInfo { SnapshotPath = snapshotPath, ExportedAtUtc = DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture), - UnityVersion = $"format:{decoded.FormatVersion}", + UnityVersion = unityVersion, + SnapFormatVersion = decoded.FormatVersion, + SessionGuid = captureMeta.SessionGuid, + ProductName = captureMeta.ProductName ?? string.Empty, + Platform = captureMeta.Platform ?? string.Empty, + RecordDateUtc = decoded.RecordDateTicksUtc > 0 + ? new DateTime(decoded.RecordDateTicksUtc, DateTimeKind.Utc).ToString("O", CultureInfo.InvariantCulture) + : string.Empty, } }; ExtractNativeRoots(decoded, data.NativeRoots); ExtractMemoryRegions(decoded, data.MemoryRegions); ExtractNativeAllocations(decoded, data.NativeAllocations); - ExtractNativeObjects(decoded, data.NativeObjects); + ExtractSystemMemoryRegions(decoded, data.SystemMemoryRegions); + + var hasResident = ResidentMemoryCalculator.HasPerObjectResident(decoded); + var (rootResidentSizes, _) = MemoryMapResidentAggregator.Compute(decoded); + var rootIdToIndex = BuildRootIdToIndex(decoded); + + ExtractNativeObjects(decoded, data.NativeObjects, rootResidentSizes, rootIdToIndex, hasResident); + ApplyRootResidentSizes(data.NativeRoots, rootResidentSizes, hasResident); var managedCrawl = ManagedSnapshotCrawler.Crawl(decoded); data.ManagedObjects.AddRange(managedCrawl.ManagedObjects); ExtractConnections(decoded, managedCrawl.ManagedConnections, data.Connections); + data.SummaryMetrics = SummaryMetricsCalculator.Compute(decoded, data.ManagedObjects); ValidateStrictInMemory(data); return data; } @@ -74,18 +94,47 @@ private static void ExtractNativeRoots(DecodedSnapshot decoded, List output) + private static Dictionary BuildRootIdToIndex(DecodedSnapshot decoded) + { + var map = new Dictionary(decoded.NativeRootIds.Length); + for (var i = 0; i < decoded.NativeRootIds.Length; i++) + map[decoded.NativeRootIds[i]] = i; + return map; + } + + private static void ExtractNativeObjects( + DecodedSnapshot decoded, + List output, + ulong[] rootResidentSizes, + Dictionary rootIdToIndex, + bool hasResident) { output.Capacity = decoded.NativeObjectNames.Length; for (var i = 0; i < decoded.NativeObjectNames.Length; i++) { var typeIndex = decoded.NativeObjectTypeIndices[i]; + var address = i < decoded.NativeObjectAddresses.Length ? decoded.NativeObjectAddresses[i] : 0UL; + var rootReferenceId = i < decoded.NativeObjectRootReferenceIds.Length + ? decoded.NativeObjectRootReferenceIds[i] + : -1L; + ulong? residentSizeBytes = null; + if (hasResident && + rootReferenceId >= 1 && + rootIdToIndex.TryGetValue(rootReferenceId, out var rootIndex) && + rootIndex < rootResidentSizes.Length) + { + residentSizeBytes = rootResidentSizes[rootIndex]; + } + output.Add(new NativeObjectRow { NativeObjectIndex = i, InstanceId = decoded.NativeObjectInstanceIds[i].ToString(CultureInfo.InvariantCulture), Name = decoded.NativeObjectNames[i] ?? string.Empty, SizeBytes = decoded.NativeObjectSizes[i], + NativeObjectAddress = address, + RootReferenceId = rootReferenceId, + ResidentSizeBytes = residentSizeBytes, TypeIndex = typeIndex, NativeTypeName = typeIndex >= 0 && typeIndex < decoded.NativeTypeNames.Length ? decoded.NativeTypeNames[typeIndex] ?? string.Empty @@ -95,6 +144,36 @@ private static void ExtractNativeObjects(DecodedSnapshot decoded, List roots, ulong[] rootResidentSizes, bool hasResident) + { + if (!hasResident) + return; + + for (var i = 0; i < roots.Count; i++) + { + var row = roots[i]; + row.ResidentSizeBytes = i < rootResidentSizes.Length ? rootResidentSizes[i] : 0UL; + roots[i] = row; + } + } + + private static void ExtractSystemMemoryRegions(DecodedSnapshot decoded, List output) + { + output.Capacity = decoded.SystemMemoryRegionAddresses.Length; + for (var i = 0; i < decoded.SystemMemoryRegionAddresses.Length; i++) + { + output.Add(new SystemMemoryRegionRow + { + RegionIndex = i, + Address = decoded.SystemMemoryRegionAddresses[i], + SizeBytes = i < decoded.SystemMemoryRegionSizes.Length ? decoded.SystemMemoryRegionSizes[i] : 0, + ResidentBytes = i < decoded.SystemMemoryRegionResidentSizes.Length ? decoded.SystemMemoryRegionResidentSizes[i] : 0, + Type = i < decoded.SystemMemoryRegionTypes.Length ? decoded.SystemMemoryRegionTypes[i] : 0, + Name = i < decoded.SystemMemoryRegionNames.Length ? decoded.SystemMemoryRegionNames[i] ?? string.Empty : string.Empty, + }); + } + } + private static void ExtractMemoryRegions(DecodedSnapshot decoded, List output) { output.Capacity = decoded.NativeMemoryRegionAddressBases.Length; @@ -126,6 +205,9 @@ private static void ExtractNativeAllocations(DecodedSnapshot decoded, List +/// Computes the Unity Memory Profiler "Summary" page metrics (Allocated Memory Distribution and +/// Managed Heap Utilization) from a decoded snapshot. +/// +/// This replicates EntriesMemoryMapCache (build/sort/post-process of an address spectrum), +/// GetPointType classification, and the post-processing in AllMemorySummaryModelBuilder / +/// ManagedMemorySummaryModelBuilder (legacy untracked fallback, graphics estimation, VM root +/// reassignment) so the tool's numbers can be validated against golden values exported from the Editor. +/// +public static class SummaryMetricsCalculator +{ + // Category labels — must match Memory Profiler's SummaryTextContent (golden trims the trailing '*' from Untracked). + private const string CategoryNative = "Native"; + private const string CategoryManaged = "Managed"; + private const string CategoryExecutablesAndMapped = "Executables & Mapped"; + private const string CategoryGraphics = "Graphics (Estimated)"; + private const string CategoryUntracked = "Untracked"; + private const string CategoryVirtualMachine = "Virtual Machine"; + private const string CategoryObjects = "Objects"; + private const string CategoryEmptyHeapSpace = "Empty Heap Space"; + + // VM native root object names, matching Memory Profiler's NativeRootReferenceEntriesCache.k_VMRootNames. + private static readonly string[] VmRootNames = ["Mono VM", "IL2CPP VM", "IL2CPPMemoryAllocator"]; + + // Source kinds; byte values match Memory Profiler's CachedSnapshot.SourceIndex.SourceId ordering, + // which the address-point sort tie-break depends on. + private enum SourceKind : byte + { + None = 0, + SystemMemoryRegion = 1, + NativeMemoryRegion = 2, + NativeAllocation = 3, + ManagedHeapSection = 4, + NativeObject = 5, + ManagedObject = 6, + } + + // OS system-memory region types, matching SystemMemoryRegionEntriesCache.MemoryType. + private enum SystemRegionMemoryType + { + Private = 0, + Mapped = 1, + Shared = 2, + Device = 3, + } + + private enum PointType + { + Free, + Untracked, + NativeReserved, + Native, + ManagedReserved, + Managed, + Device, + Mapped, + Shared, + AndroidRuntime, + } + + private struct Mem + { + public ulong Committed; + public ulong Resident; + } + + private struct AddressPoint + { + public ulong Address; + public long PairId; + public SourceKind Kind; + public int Index; + public bool IsEnd; + } + + /// + /// Computes summary metrics from a decoded snapshot and its crawled managed objects. + /// + public static SummaryMetrics Compute( + DecodedSnapshot decoded, + IReadOnlyList managedObjects) + { + var hasSystemRegions = decoded.SystemMemoryRegionAddresses.Length > 0; + var hasResidentPages = ResidentMemoryCalculator.HasPerObjectResident(decoded); + var isAndroid = (decoded.CaptureMetadata.Platform ?? string.Empty) + .Contains("Android", StringComparison.OrdinalIgnoreCase); + + var points = BuildSortedPoints(decoded, managedObjects); + PostProcess(points); + + var pageStates = hasResidentPages ? new BitArray(decoded.SystemMemoryResidentPageStates[0]) : null; + var pageSize = decoded.SystemMemoryResidentPageSize; + + var native = default(Mem); + var managed = default(Mem); + var mapped = default(Mem); + var graphics = default(Mem); + var untracked = default(Mem); + var androidRuntime = default(Mem); + + var vmSection = default(Mem); + var emptyHeapSpace = default(Mem); + var objects = default(Mem); + + // The VM root's size is the map-resolved committed/resident of the native allocations and objects + // rooted to it (matching ProcessedNativeRoots), not the raw NativeRootReferences accumulated size. + var vmRootIndex = FindVmRootIndex(decoded); + var vmRootId = vmRootIndex >= 0 && vmRootIndex < decoded.NativeRootIds.Length + ? decoded.NativeRootIds[vmRootIndex] + : long.MinValue; + var vmRoot = default(Mem); + + var currentSystemRegion = -1; + for (var i = 0; i < points.Length - 1; i++) + { + var cur = points[i]; + if (cur.Kind == SourceKind.SystemMemoryRegion) + currentSystemRegion = cur.Index; + else if (cur.Kind == SourceKind.None) + currentSystemRegion = -1; + + if (cur.Kind == SourceKind.None) + continue; + + var size = points[i + 1].Address - cur.Address; + if (size == 0) + continue; + + var resident = 0UL; + if (hasSystemRegions) + { + // Items outside any system region exist due to capture timing differences; skip them. + if (currentSystemRegion < 0) + continue; + + if (pageStates != null && CanComputeResident(decoded, currentSystemRegion)) + { + resident = ResidentMemoryCalculator.CalculateResidentForRange( + decoded, pageStates, pageSize, currentSystemRegion, cur.Address, size); + } + } + + var span = new Mem { Committed = size, Resident = resident }; + if (vmRootId != long.MinValue && SpanRootReferenceId(decoded, cur) == vmRootId) + Add(ref vmRoot, span); + + switch (GetPointType(decoded, cur, isAndroid)) + { + case PointType.Native: + case PointType.NativeReserved: + Add(ref native, span); + break; + case PointType.Managed: + case PointType.ManagedReserved: + Add(ref managed, span); + AddManagedBreakdown(decoded, cur, span, ref vmSection, ref emptyHeapSpace, ref objects); + break; + case PointType.Mapped: + Add(ref mapped, span); + break; + case PointType.Device: + Add(ref graphics, span); + break; + case PointType.Shared: + case PointType.Untracked: + Add(ref untracked, span); + break; + case PointType.AndroidRuntime: + Add(ref androidRuntime, span); + break; + } + } + + var total = default(Mem); + Add(ref total, native); + Add(ref total, managed); + Add(ref total, mapped); + Add(ref total, graphics); + Add(ref total, untracked); + Add(ref total, androidRuntime); + + ApplyTargetStatHeuristics(decoded, ref total, ref graphics, ref untracked); + + // Move the VM root out of Native into Managed (Allocated Memory Distribution), mirroring the builder. + var vmRootCommitted = vmRoot.Committed; + var vmRootResident = vmRoot.Resident; + if (vmRootId != long.MinValue) + { + managed.Committed += vmRootCommitted; + managed.Resident += vmRootResident; + native.Committed -= Math.Min(native.Committed, vmRootCommitted); + native.Resident -= Math.Min(native.Resident, vmRootResident); + } + + var result = new SummaryMetrics + { + // Total committed comes from the address-spectrum total (or legacy fallback); graphics/untracked + // shuffling preserves it. Total resident is the resident of EVERY flattened span (including the + // Graphics and Untracked regions), matching ResidentMemorySummaryModelBuilder — not just the rows + // whose per-category resident is surfaced in the UI. + TotalAllocatedBytes = total.Committed, + TotalResidentBytes = total.Resident, + }; + + result.AllocatedMemoryDistribution.Add(Category(CategoryNative, native, true)); + result.AllocatedMemoryDistribution.Add(Category(CategoryManaged, managed, true)); + result.AllocatedMemoryDistribution.Add(Category(CategoryExecutablesAndMapped, mapped, true)); + result.AllocatedMemoryDistribution.Add(Category(CategoryGraphics, graphics, false)); + result.AllocatedMemoryDistribution.Add(Category(CategoryUntracked, untracked, false)); + if (androidRuntime.Committed > 0) + result.AllocatedMemoryDistribution.Add(Category("Android Runtime", androidRuntime, true)); + + // Managed Heap Utilization: add the VM root to the Virtual Machine row, mirroring the builder. + var virtualMachine = vmSection; + virtualMachine.Committed += vmRootCommitted; + virtualMachine.Resident += vmRootResident; + + result.ManagedHeapUtilization.Add(Category(CategoryVirtualMachine, virtualMachine, true)); + result.ManagedHeapUtilization.Add(Category(CategoryObjects, objects, true)); + result.ManagedHeapUtilization.Add(Category(CategoryEmptyHeapSpace, emptyHeapSpace, true)); + + return result; + } + + private static void AddManagedBreakdown( + DecodedSnapshot decoded, + AddressPoint cur, + Mem span, + ref Mem vmSection, + ref Mem emptyHeapSpace, + ref Mem objects) + { + switch (cur.Kind) + { + case SourceKind.ManagedHeapSection: + if (cur.Index < decoded.ManagedHeapSectionTypes.Length && + decoded.ManagedHeapSectionTypes[cur.Index] == ManagedHeapSectionKind.VirtualMachine) + Add(ref vmSection, span); + else + Add(ref emptyHeapSpace, span); + break; + case SourceKind.ManagedObject: + Add(ref objects, span); + break; + } + } + + /// + /// Legacy untracked/total fallback and graphics estimation, ported from + /// AllMemorySummaryModelBuilder.BuildSummary. + /// + private static void ApplyTargetStatHeuristics(DecodedSnapshot decoded, ref Mem total, ref Mem graphics, ref Mem untracked) + { + var stats = decoded.TargetMemoryStats; + if (stats == null) + return; + + var hasSystemRegions = decoded.SystemMemoryRegionAddresses.Length > 0; + if (!hasSystemRegions && stats.TotalVirtualMemory > 0) + { + untracked = new Mem + { + Committed = stats.TotalVirtualMemory > total.Committed ? stats.TotalVirtualMemory - total.Committed : 0, + }; + total = new Mem { Committed = stats.TotalVirtualMemory }; + } + + if (graphics.Committed < stats.GraphicsUsedMemory) + { + // System regions under-report graphics; reassign from untracked (capped by what untracked has). + var delta = Math.Min(stats.GraphicsUsedMemory - graphics.Committed, untracked.Committed); + untracked = new Mem { Committed = untracked.Committed - delta }; + graphics = new Mem { Committed = graphics.Committed + delta }; + } + else + { + // Note: consoles with UseDeviceMemoryForGraphics fully track GPU memory and skip this branch. + // The tool does not yet decode native-region GPU allocator indices, so we always apply it. + var untrackedGraphics = graphics.Committed - stats.GraphicsUsedMemory; + untracked = new Mem { Committed = untracked.Committed + untrackedGraphics }; + graphics = new Mem { Committed = graphics.Committed - untrackedGraphics }; + } + } + + /// + /// Root reference id of the native allocation or native object owning a flattened span, or null for + /// other source kinds. Used to attribute spans to the VM root. + /// + private static long? SpanRootReferenceId(DecodedSnapshot decoded, AddressPoint point) + { + switch (point.Kind) + { + case SourceKind.NativeAllocation: + return point.Index < decoded.NativeAllocationRootReferenceIds.Length + ? decoded.NativeAllocationRootReferenceIds[point.Index] + : null; + case SourceKind.NativeObject: + return point.Index < decoded.NativeObjectRootReferenceIds.Length + ? decoded.NativeObjectRootReferenceIds[point.Index] + : null; + default: + return null; + } + } + + private static int FindVmRootIndex(DecodedSnapshot decoded) + { + for (var i = 0; i < decoded.NativeRootObjectNames.Length; i++) + { + var name = decoded.NativeRootObjectNames[i]; + if (string.IsNullOrEmpty(name)) + continue; + + foreach (var vmRootName in VmRootNames) + { + if (string.Equals(name, vmRootName, StringComparison.Ordinal)) + return i; + } + } + + return -1; + } + + private static PointType GetPointType(DecodedSnapshot decoded, AddressPoint point, bool isAndroid) + { + switch (point.Kind) + { + case SourceKind.None: + return PointType.Free; + + case SourceKind.SystemMemoryRegion: + { + if (isAndroid && point.Index < decoded.SystemMemoryRegionNames.Length) + { + var name = decoded.SystemMemoryRegionNames[point.Index] ?? string.Empty; + if (name.StartsWith("[anon:dalvik-", StringComparison.Ordinal) || + name.StartsWith("/dev/ashmem/dalvik-", StringComparison.Ordinal)) + return PointType.AndroidRuntime; + if (name.StartsWith("/dev/", StringComparison.Ordinal)) + return PointType.Device; + } + + var regionType = point.Index < decoded.SystemMemoryRegionTypes.Length + ? (SystemRegionMemoryType)decoded.SystemMemoryRegionTypes[point.Index] + : SystemRegionMemoryType.Private; + return regionType switch + { + SystemRegionMemoryType.Device => PointType.Device, + SystemRegionMemoryType.Mapped => PointType.Mapped, + SystemRegionMemoryType.Shared => PointType.Shared, + _ => PointType.Untracked, + }; + } + + case SourceKind.NativeMemoryRegion: + return PointType.NativeReserved; + + case SourceKind.NativeAllocation: + case SourceKind.NativeObject: + return PointType.Native; + + case SourceKind.ManagedHeapSection: + return point.Index < decoded.ManagedHeapSectionTypes.Length && + decoded.ManagedHeapSectionTypes[point.Index] == ManagedHeapSectionKind.VirtualMachine + ? PointType.Managed + : PointType.ManagedReserved; + + case SourceKind.ManagedObject: + return PointType.Managed; + + default: + return PointType.Free; + } + } + + private static AddressPoint[] BuildSortedPoints(DecodedSnapshot decoded, IReadOnlyList managedObjects) + { + var points = new List( + (decoded.SystemMemoryRegionAddresses.Length + + decoded.NativeMemoryRegionAddressBases.Length + + decoded.ManagedHeapSectionStartAddresses.Length + + decoded.NativeAllocationAddresses.Length + + decoded.NativeObjectAddresses.Length + + managedObjects.Count) * 2); + + var pairId = 0L; + + for (var i = 0; i < decoded.SystemMemoryRegionAddresses.Length; i++) + AddPair(points, ref pairId, SourceKind.SystemMemoryRegion, i, decoded.SystemMemoryRegionAddresses[i], decoded.SystemMemoryRegionSizes[i]); + + for (var i = 0; i < decoded.NativeMemoryRegionAddressBases.Length; i++) + { + var address = decoded.NativeMemoryRegionAddressBases[i]; + var size = decoded.NativeMemoryRegionAddressSizes[i]; + var name = i < decoded.NativeMemoryRegionNames.Length ? decoded.NativeMemoryRegionNames[i] ?? string.Empty : string.Empty; + // Exclude "virtual" allocators which report non-committed memory (matches Memory Profiler). + if (address == 0 || name.Contains("Virtual Memory", StringComparison.Ordinal)) + continue; + AddPair(points, ref pairId, SourceKind.NativeMemoryRegion, i, address, size); + } + + for (var i = 0; i < decoded.ManagedHeapSectionStartAddresses.Length; i++) + { + var size = i < decoded.ManagedHeapSectionBytes.Length ? (ulong)decoded.ManagedHeapSectionBytes[i].Length : 0; + AddPair(points, ref pairId, SourceKind.ManagedHeapSection, i, decoded.ManagedHeapSectionStartAddresses[i], size); + } + + for (var i = 0; i < decoded.NativeAllocationAddresses.Length; i++) + AddPair(points, ref pairId, SourceKind.NativeAllocation, i, decoded.NativeAllocationAddresses[i], decoded.NativeAllocationSizes[i]); + + for (var i = 0; i < decoded.NativeObjectAddresses.Length; i++) + AddPair(points, ref pairId, SourceKind.NativeObject, i, decoded.NativeObjectAddresses[i], decoded.NativeObjectSizes[i]); + + for (var i = 0; i < managedObjects.Count; i++) + { + var row = managedObjects[i]; + AddPair(points, ref pairId, SourceKind.ManagedObject, i, row.Address, row.SizeBytes > 0 ? (ulong)row.SizeBytes : 0); + } + + var array = points.ToArray(); + Array.Sort(array, Compare); + return array; + } + + private static void AddPair(List points, ref long pairId, SourceKind kind, int index, ulong address, ulong size) + { + if (size == 0) + return; + + var id = pairId++; + points.Add(new AddressPoint { Address = address, PairId = id, Kind = kind, Index = index, IsEnd = false }); + points.Add(new AddressPoint { Address = address + size, PairId = id, Kind = kind, Index = index, IsEnd = true }); + } + + /// + /// Address-point ordering ported from EntriesMemoryMapCache.AddressPoint.CompareTo: + /// by address, then end-before-start, then by source kind (descending for end points). + /// + private static int Compare(AddressPoint a, AddressPoint b) + { + var cmp = a.Address.CompareTo(b.Address); + if (cmp != 0) + return cmp; + + if (a.IsEnd != b.IsEnd) + return a.IsEnd ? -1 : 1; + + var kindCmp = ((byte)a.Kind).CompareTo((byte)b.Kind); + return a.IsEnd ? -kindCmp : kindCmp; + } + + /// + /// Resolves the owning source of every flattened span by walking the sorted points with a hierarchy + /// stack, rewriting end points to continue their parent (or free space). Ported from + /// EntriesMemoryMapCache.PostProcess; child-count bookkeeping is omitted as the flat scan only + /// reads each point's resolved source. + /// + private static void PostProcess(AddressPoint[] points) + { + var stack = new List(16); + for (var i = 0; i < points.Length; i++) + { + var point = points[i]; + if (point.IsEnd) + { + if (stack.Count == 0) + { + SetFree(ref points[i]); + continue; + } + + var startIdx = stack[^1]; + if (points[startIdx].PairId != point.PairId) + { + var level = FindStackLevel(points, stack, point.PairId); + if (level < 0) + { + // No matching start: treat as a continuation of the previous span. + points[i].Kind = points[i - 1].Kind; + points[i].Index = points[i - 1].Index; + continue; + } + + stack.RemoveRange(level, stack.Count - level); + } + else + { + stack.RemoveAt(stack.Count - 1); + } + + if (stack.Count > 0) + { + var parent = points[stack[^1]]; + points[i].Kind = parent.Kind; + points[i].Index = parent.Index; + } + else + { + SetFree(ref points[i]); + } + } + else + { + if (stack.Count > 0 && points[stack[^1]].Kind == point.Kind) + { + // Same-type nesting indicates faulty/overlapping data; drop the enclosing point. + stack.RemoveAt(stack.Count - 1); + } + + stack.Add(i); + } + } + } + + private static int FindStackLevel(AddressPoint[] points, List stack, long pairId) + { + for (var level = 0; level < stack.Count; level++) + { + if (points[stack[level]].PairId == pairId) + return level; + } + + return -1; + } + + private static void SetFree(ref AddressPoint point) + { + point.Kind = SourceKind.None; + point.Index = -1; + } + + private static bool CanComputeResident(DecodedSnapshot decoded, int regionIndex) => + regionIndex >= 0 && + regionIndex < decoded.SystemMemoryResidentPageAddresses.Length && + regionIndex < decoded.SystemMemoryResidentPageFirstIndices.Length && + regionIndex < decoded.SystemMemoryResidentPageLastIndices.Length; + + private static void Add(ref Mem target, Mem value) + { + target.Committed += value.Committed; + target.Resident += value.Resident; + } + + private static SummaryCategory Category(string name, Mem mem, bool residentAvailable) => + new() + { + Name = name, + CommittedBytes = mem.Committed, + ResidentBytes = residentAvailable ? mem.Resident : 0, + ResidentAvailable = residentAvailable, + }; +} diff --git a/Core/Report/MultiSnapshotReport/MultiSnapshotHtmlRenderer.cs b/Core/Report/MultiSnapshotReport/MultiSnapshotHtmlRenderer.cs new file mode 100644 index 0000000..3e21f49 --- /dev/null +++ b/Core/Report/MultiSnapshotReport/MultiSnapshotHtmlRenderer.cs @@ -0,0 +1,315 @@ +using System.Globalization; +using System.Text; +using MemorySnapshotDataTools.Report; + +namespace MemorySnapshotDataTools.Report.MultiSnapshotReport; + +/// +/// Renders a to a self-contained HTML document with one session-grouped table. +/// +public static class MultiSnapshotHtmlRenderer +{ + private const int ColSnapshot = 0; + private const int ColAbCount = 1; + private const int ColAbAlloc = 2; + private const int ColAbRes = 3; + private const int ColSfCount = 4; + private const int ColSfAlloc = 5; + private const int ColSfRes = 6; + private const int ColPmrAlloc = 7; + private const int ColPmrRes = 8; + private const int ColumnCount = 9; + + /// + /// Builds the full HTML report string from the model. + /// + public static string Render(MultiSnapshotReportModel model) + { + var sb = new StringBuilder(); + sb.Append(""" + + + + + + + """); + sb.Append(Escape(model.Title)); + sb.Append(""" + + + + +
+

+ """); + sb.Append(Escape(model.Title)); + sb.Append("

\n

"); + sb.Append(Escape(model.SourceDirectory)); + sb.Append(" · Generated "); + sb.Append(Escape(model.GeneratedAtUtc)); + sb.Append(" · "); + sb.Append(model.Sessions.Sum(s => s.Snapshots.Count).ToString(CultureInfo.InvariantCulture)); + sb.Append(" snapshots

\n"); + + if (model.Sessions.Count == 0) + sb.Append("

No matching database files found.

"); + else + sb.Append(RenderUnifiedTable(model)); + + sb.Append(""" +
+ + + + """); + return sb.ToString(); + } + + private static string RenderUnifiedTable(MultiSnapshotReportModel model) + { + var sb = new StringBuilder(); + sb.Append(""" +
+ + + + + + + + + + + + + + + + + + + + """); + + foreach (var session in model.Sessions) + { + sb.Append("\n"); + + foreach (var snap in session.Snapshots) + { + var ab = GetTypeMetrics(snap, "AssetBundle"); + var sf = GetTypeMetrics(snap, "SerializedFile"); + var pmr = AggregateRemapper(snap); + + sb.Append(""); + AppendSnapshotCell(sb, snap); + AppendCountCell(sb, ColAbCount, ab.Count); + AppendBytesCell(sb, ColAbAlloc, ab.AllocatedBytes); + AppendBytesCell(sb, ColAbRes, ab.ResidentBytes); + AppendCountCell(sb, ColSfCount, sf.Count); + AppendBytesCell(sb, ColSfAlloc, sf.AllocatedBytes); + AppendBytesCell(sb, ColSfRes, sf.ResidentBytes); + AppendBytesCell(sb, ColPmrAlloc, pmr.AllocatedBytes); + AppendBytesCell(sb, ColPmrRes, pmr.ResidentBytes); + sb.Append("\n"); + } + } + + sb.Append("
SnapshotAsset BundleSerialized FilePMR
CountAllocatedResidentCountAllocatedResidentAllocatedResident
"); + sb.Append(PlatformIconHtml.Render(session.PlatformKind)); + sb.Append(' '); + sb.Append(Escape(session.DisplayTitle)); + sb.Append(" ("); + sb.Append(session.Snapshots.Count.ToString(CultureInfo.InvariantCulture)); + sb.Append(" snapshot"); + if (session.Snapshots.Count != 1) + sb.Append('s'); + sb.Append(")
"); + return sb.ToString(); + } + + private static void AppendSnapshotCell(StringBuilder sb, SnapshotMetricsRow snap) + { + sb.Append(""); + sb.Append(PlatformIconHtml.Render(snap.PlatformKind, snap.Platform)); + sb.Append(""); + sb.Append(Escape(snap.SnapshotName)); + sb.Append(""); + } + + private static void AppendCountCell(StringBuilder sb, int col, int count) + { + sb.Append(""); + sb.Append(count.ToString("N0", CultureInfo.InvariantCulture)); + sb.Append(""); + } + + private static void AppendBytesCell(StringBuilder sb, int col, long bytes) + { + sb.Append(""); + sb.Append(ReportHtmlHelper.FmtBytesHtml(bytes)); + sb.Append(""); + } + + private static void AppendBytesCell(StringBuilder sb, int col, long? bytes) + { + sb.Append(""); + sb.Append(ReportHtmlHelper.FmtBytesHtml(bytes.Value)); + } + else + { + sb.Append(">N/A"); + } + + sb.Append(""); + } + + private static NativeTypeSnapshotMetrics GetTypeMetrics(SnapshotMetricsRow snap, string typeName) => + snap.NativeTypes.TryGetValue(typeName, out var m) + ? m + : new NativeTypeSnapshotMetrics { NativeTypeName = typeName }; + + private static NativeRootSnapshotMetrics AggregateRemapper(SnapshotMetricsRow snap) + { + if (snap.RemapperRoots.Count == 0) + return new NativeRootSnapshotMetrics(); + + long alloc = 0; + long? resident = 0; + var hasResident = true; + foreach (var root in snap.RemapperRoots) + { + alloc += root.AllocatedBytes; + if (root.ResidentBytes.HasValue) + resident = (resident ?? 0) + root.ResidentBytes.Value; + else + hasResident = false; + } + + return new NativeRootSnapshotMetrics + { + AllocatedBytes = alloc, + ResidentBytes = hasResident ? resident : null, + }; + } + + private static string Escape(string? value) => + System.Net.WebUtility.HtmlEncode(value ?? string.Empty); + + private static string EscapeAttr(string value) => + System.Net.WebUtility.HtmlEncode(value); + + private const string SortableScript = """ + document.querySelectorAll('table.multi-snapshot th.sub[data-col]').forEach(function(th) { + th.style.cursor = 'pointer'; + th.addEventListener('click', function() { + var table = th.closest('table'); + var col = th.getAttribute('data-col'); + var tbody = table.querySelector('tbody'); + var dir = th.dataset.sortDir === 'asc' ? -1 : 1; + table.querySelectorAll('th.sub[data-col]').forEach(function(h) { delete h.dataset.sortDir; }); + th.dataset.sortDir = dir === 1 ? 'asc' : 'desc'; + var blocks = []; + var current = null; + tbody.querySelectorAll('tr').forEach(function(tr) { + if (tr.classList.contains('session-header')) { + current = { header: tr, rows: [] }; + blocks.push(current); + } else if (tr.classList.contains('snapshot-row') && current) { + current.rows.push(tr); + } + }); + blocks.forEach(function(block) { + block.rows.sort(function(a, b) { + var ac = a.querySelector('td[data-col="' + col + '"]'); + var bc = b.querySelector('td[data-col="' + col + '"]'); + var av = ac ? (ac.dataset.sort || ac.textContent.trim()) : ''; + var bv = bc ? (bc.dataset.sort || bc.textContent.trim()) : ''; + var an = parseFloat(String(av).replace(/,/g, '')); + var bn = parseFloat(String(bv).replace(/,/g, '')); + if (!isNaN(an) && !isNaN(bn)) return dir * (an - bn); + return dir * String(av).localeCompare(String(bv)); + }); + tbody.appendChild(block.header); + block.rows.forEach(function(r) { tbody.appendChild(r); }); + }); + }); + }); + """; + + private static readonly string Css = """ + *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } + body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; font-size: 13px; background: #f0f2f5; color: #1a1a2e; padding: 24px; line-height: 1.5; } + main { max-width: 100%; margin: 0 auto; } + h1 { font-size: 22px; font-weight: 700; margin-bottom: 4px; } + .subtitle { font-size: 12px; color: #666; margin-bottom: 24px; font-family: "SF Mono", Consolas, monospace; word-break: break-all; } + .table-wrap { background: #fff; border-radius: 8px; box-shadow: 0 1px 4px rgba(0,0,0,.08); overflow-x: auto; overflow-y: visible; } + table.multi-snapshot { width: 100%; border-collapse: separate; border-spacing: 0; table-layout: auto; --header-row1-h: 37px; } + thead th { + position: sticky; + background: #1a1a2e; + color: #fff; + font-size: 11px; + font-weight: 600; + padding: 8px 10px; + text-align: left; + white-space: nowrap; + border: 1px solid #2d2d44; + box-shadow: 0 1px 0 #2d2d44; + } + thead tr:first-child th { top: 0; z-index: 3; } + thead tr:nth-child(2) th { top: var(--header-row1-h); z-index: 2; } + thead th.col-snapshot { vertical-align: bottom; z-index: 4; } + thead th.group-hdr { text-align: center; text-transform: uppercase; letter-spacing: 0.03em; } + thead th.sub { text-transform: uppercase; font-size: 10px; font-weight: 500; } + thead th.num, td.num { text-align: right; } + tbody tr.session-header td { background: #e8ecf4; font-weight: 600; font-size: 12px; padding: 10px 12px; border-top: 2px solid #c5cee0; border-bottom: 1px solid #c5cee0; } + tbody tr.session-header:first-child td { border-top: none; } + .session-count { font-weight: 400; color: #666; } + tbody tr.snapshot-row:nth-child(even) { background: #f8f9fb; } + tbody tr.snapshot-row:hover { background: #eef2ff; } + td { padding: 6px 10px; border-bottom: 1px solid #f0f2f5; vertical-align: top; } + .platform-icon { display: inline-flex; align-items: center; margin-right: 6px; vertical-align: middle; color: #475569; } + .platform-icon.ios { color: #1a1a2e; } + .platform-icon.android { color: #3ddc84; } + td.snapshot-name { font-size: 11px; white-space: nowrap; min-width: 280px; } + .snapshot-label { display: inline-flex; align-items: center; gap: 4px; } + .snapshot-filename { font-family: "SF Mono", Consolas, monospace; } + td.num { font-variant-numeric: tabular-nums; font-family: "SF Mono", Consolas, monospace; font-size: 12px; white-space: nowrap; } + .bytes { border-bottom: 1px dotted #94a3b8; cursor: help; } + em.na { color: #999; font-style: italic; } + .empty { color: #999; font-style: italic; padding: 24px; } + """; +} diff --git a/Core/Report/MultiSnapshotReport/MultiSnapshotReportBuilder.cs b/Core/Report/MultiSnapshotReport/MultiSnapshotReportBuilder.cs new file mode 100644 index 0000000..7e808f8 --- /dev/null +++ b/Core/Report/MultiSnapshotReport/MultiSnapshotReportBuilder.cs @@ -0,0 +1,422 @@ +using System.Globalization; +using DuckDB.NET.Data; +using MemorySnapshotDataTools.Parser; +using MemorySnapshotDataTools.Validation; +using Microsoft.Data.Sqlite; + +namespace MemorySnapshotDataTools.Report.MultiSnapshotReport; + +/// +/// Queries a set of DuckDB or SQLite files for memory metrics on specific native types and roots, +/// then builds a grouped by capture session. +/// +public static class MultiSnapshotReportBuilder +{ + private static readonly string[] TrackedNativeTypes = ["AssetBundle", "SerializedFile"]; + + /// + /// Scans for database files matching , + /// queries each for tracked metrics, and returns a grouped report model. + /// + /// Directory containing .duckdb or .db files. + /// Optional case-insensitive substring filter on filenames. + /// Report title. + /// Populated multi-snapshot report model. + public static MultiSnapshotReportModel Build(string directory, string? nameFilter, string title) + { + var dbPaths = Directory.EnumerateFiles(directory, "*.*", SearchOption.TopDirectoryOnly) + .Where(p => + { + var ext = Path.GetExtension(p); + return ext.Equals(".duckdb", StringComparison.OrdinalIgnoreCase) + || ext.Equals(".db", StringComparison.OrdinalIgnoreCase); + }) + .Where(p => string.IsNullOrWhiteSpace(nameFilter) + || Path.GetFileName(p).Contains(nameFilter, StringComparison.OrdinalIgnoreCase)) + .OrderBy(p => p, StringComparer.OrdinalIgnoreCase) + .ToList(); + + var rows = new List(); + foreach (var dbPath in dbPaths) + { + try + { + rows.Add(QueryDatabase(dbPath)); + } + catch (Exception ex) + { + Console.Error.WriteLine($"Skipping {Path.GetFileName(dbPath)}: {ex.Message}"); + } + } + + var sessions = MultiSnapshotSessionGrouper.BuildGroups(rows); + + return new MultiSnapshotReportModel + { + Title = title, + GeneratedAtUtc = DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm:ss", CultureInfo.InvariantCulture) + " UTC", + SourceDirectory = Path.GetFullPath(directory), + Sessions = sessions, + }; + } + + private static SnapshotMetricsRow QueryDatabase(string dbPath) + { + var ext = Path.GetExtension(dbPath); + return ext.Equals(".db", StringComparison.OrdinalIgnoreCase) + ? QuerySqlite(dbPath) + : QueryDuckDb(dbPath); + } + + private static SnapshotMetricsRow QueryDuckDb(string dbPath) + { + using var connection = new DuckDBConnection($"Data Source={dbPath}"); + connection.Open(); + + var snapshotMeta = QuerySnapshotMetadata(connection, isDuckDb: true); + var nativeTypes = QueryNativeTypes(connection, isDuckDb: true); + var remapperRoots = QueryRemapperRoots(connection, isDuckDb: true); + return BuildRow(dbPath, nativeTypes, remapperRoots, snapshotMeta); + } + + private static SnapshotMetricsRow QuerySqlite(string dbPath) + { + using var connection = new SqliteConnection($"Data Source={dbPath}"); + connection.Open(); + + var snapshotMeta = QuerySnapshotMetadata(connection, isDuckDb: false); + var nativeTypes = QueryNativeTypes(connection, isDuckDb: false); + var remapperRoots = QueryRemapperRoots(connection, isDuckDb: false); + return BuildRow(dbPath, nativeTypes, remapperRoots, snapshotMeta); + } + + private static Dictionary QueryNativeTypes(object connection, bool isDuckDb) + { + var result = new Dictionary(StringComparer.Ordinal); + foreach (var typeName in TrackedNativeTypes) + { + result[typeName] = new NativeTypeSnapshotMetrics + { + NativeTypeName = typeName, + Count = 0, + AllocatedBytes = 0, + ResidentBytes = 0, + }; + } + + var nullResidentExpr = isDuckDb ? "CAST(NULL AS BIGINT)" : "NULL"; + var objectResidentExpr = HasColumn(connection, isDuckDb, "native_objects", "resident_size_bytes") + ? "COALESCE(SUM(resident_size_bytes), 0)" + : nullResidentExpr; + var rootResidentExpr = HasColumn(connection, isDuckDb, "native_roots", "resident_size_bytes") + ? "COALESCE(SUM(resident_size_bytes), 0)" + : nullResidentExpr; + + var assetBundleSql = $""" + SELECT COUNT(*) AS obj_count, + COALESCE(SUM(size_bytes), 0) AS allocated_bytes, + {objectResidentExpr} AS resident_bytes + FROM native_objects + WHERE native_type_name = '{GoldenValidationQueries.AssetBundleNativeTypeName}' + AND is_destroyed = false + """; + + var serializedFileSql = $""" + SELECT COUNT(*) AS obj_count, + COALESCE(SUM(accumulated_size_bytes), 0) AS allocated_bytes, + {rootResidentExpr} AS resident_bytes + FROM native_roots + WHERE {GoldenValidationQueries.SerializedFileAreaPredicate} + """; + + ReadNativeTypeAggregate(connection, isDuckDb, assetBundleSql, GoldenValidationQueries.AssetBundleNativeTypeName, result); + ReadNativeTypeAggregate(connection, isDuckDb, serializedFileSql, GoldenValidationQueries.SerializedFileMetricName, result); + return result; + } + + private static bool HasColumn(object connection, bool isDuckDb, string tableName, string columnName) + { + var sql = isDuckDb + ? $"SELECT 1 FROM pragma_table_info('{tableName}') WHERE name = '{columnName}' LIMIT 1" + : $"SELECT 1 FROM pragma_table_info('{tableName}') WHERE name = '{columnName}' LIMIT 1"; + + if (isDuckDb) + { + using var cmd = ((DuckDBConnection)connection).CreateCommand(); + cmd.CommandText = sql; + return cmd.ExecuteScalar() != null; + } + + using var sqliteCmd = ((SqliteConnection)connection).CreateCommand(); + sqliteCmd.CommandText = sql; + return sqliteCmd.ExecuteScalar() != null; + } + + private static void ReadNativeTypeAggregate( + object connection, + bool isDuckDb, + string sql, + string typeName, + Dictionary result) + { + if (isDuckDb) + { + using var cmd = ((DuckDBConnection)connection).CreateCommand(); + cmd.CommandText = sql; + using var reader = cmd.ExecuteReader(); + if (!reader.Read()) + return; + + result[typeName] = new NativeTypeSnapshotMetrics + { + NativeTypeName = typeName, + Count = DbScalarReader.GetInt32(reader, 0), + AllocatedBytes = DbScalarReader.GetInt64(reader, 1), + ResidentBytes = reader.IsDBNull(2) ? null : DbScalarReader.GetInt64(reader, 2), + }; + return; + } + + using var sqliteCmd = ((SqliteConnection)connection).CreateCommand(); + sqliteCmd.CommandText = sql; + using var sqliteReader = sqliteCmd.ExecuteReader(); + if (!sqliteReader.Read()) + return; + + result[typeName] = new NativeTypeSnapshotMetrics + { + NativeTypeName = typeName, + Count = sqliteReader.GetInt32(0), + AllocatedBytes = sqliteReader.GetInt64(1), + ResidentBytes = sqliteReader.IsDBNull(2) ? null : sqliteReader.GetInt64(2), + }; + } + + private static List QueryRemapperRoots(object connection, bool isDuckDb) + { + var hasRootResident = HasColumn(connection, isDuckDb, "native_roots", "resident_size_bytes"); + var residentSelect = hasRootResident + ? "resident_size_bytes" + : isDuckDb ? "CAST(NULL AS BIGINT) AS resident_size_bytes" : "NULL AS resident_size_bytes"; + var sql = $""" + SELECT area_name, object_name, + accumulated_size_bytes AS allocated_bytes, + {residentSelect} + FROM native_roots + WHERE object_name LIKE '%Remapper%' + OR (COALESCE(area_name, '') || ':' || COALESCE(object_name, '')) LIKE '%PersistentManager%Remapper%' + """; + + var roots = new List(); + if (isDuckDb) + { + using var cmd = ((DuckDBConnection)connection).CreateCommand(); + cmd.CommandText = sql; + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + roots.Add(ReadRootRow(reader)); + } + } + else + { + using var cmd = ((SqliteConnection)connection).CreateCommand(); + cmd.CommandText = sql.Replace("COALESCE", "IFNULL", StringComparison.Ordinal); + using var reader = cmd.ExecuteReader(); + while (reader.Read()) + { + roots.Add(ReadRootRow(reader)); + } + } + + return roots; + } + + private static NativeRootSnapshotMetrics ReadRootRow(System.Data.Common.DbDataReader reader) => + new() + { + AreaName = reader.IsDBNull(0) ? string.Empty : reader.GetString(0), + ObjectName = reader.IsDBNull(1) ? string.Empty : reader.GetString(1), + AllocatedBytes = DbScalarReader.GetInt64(reader, 2), + ResidentBytes = reader.IsDBNull(3) ? null : DbScalarReader.GetInt64(reader, 3), + }; + + private static SnapshotMetricsRow BuildRow( + string dbPath, + Dictionary nativeTypes, + List remapperRoots, + DbSnapshotMetadata dbMeta) + { + var fileName = Path.GetFileNameWithoutExtension(dbPath); + var meta = EnrichMetadata(dbPath, fileName, dbMeta); + var filenameSession = MultiSnapshotSessionKey.FromFileName(fileName, meta.UnityVersionDisplay); + + var row = new SnapshotMetricsRow + { + SnapshotName = fileName, + DatabasePath = dbPath, + SessionKey = filenameSession.SessionKey, + CaptureDate = filenameSession.CaptureDate, + UnityVersion = meta.UnityVersionDisplay, + SnapFormatVersion = meta.SnapFormatVersion, + SessionGuid = meta.SessionGuid, + ProductName = meta.ProductName, + Platform = meta.Platform, + PlatformKind = CapturePlatformKindExtensions.FromPlatformName(meta.Platform), + SortTimestamp = meta.SortTimestamp, + NativeTypes = nativeTypes, + RemapperRoots = remapperRoots, + }; + + row = row with { SessionKey = MultiSnapshotSessionGrouper.BuildClusterKey(row) }; + return row; + } + + private static EnrichedSnapshotMetadata EnrichMetadata(string dbPath, string fileName, DbSnapshotMetadata dbMeta) + { + var needsSnap = dbMeta.SessionGuid == 0 + || string.IsNullOrWhiteSpace(dbMeta.Platform) + || string.IsNullOrWhiteSpace(dbMeta.ProductName) + || (string.IsNullOrWhiteSpace(dbMeta.UnityVersion) + || dbMeta.UnityVersion.StartsWith("format:", StringComparison.OrdinalIgnoreCase)); + + CaptureMetadata? snapMeta = null; + if (needsSnap) + { + var snapPath = Path.ChangeExtension(dbPath, ".snap"); + if (File.Exists(snapPath)) + { + try + { + snapMeta = SnapMetadataReader.Read(snapPath); + } + catch (Exception ex) + { + Console.Error.WriteLine($"Metadata read failed for {Path.GetFileName(snapPath)}: {ex.Message}"); + } + } + } + + var sessionGuid = dbMeta.SessionGuid != 0 ? dbMeta.SessionGuid : snapMeta?.SessionGuid ?? 0; + var productName = !string.IsNullOrWhiteSpace(dbMeta.ProductName) ? dbMeta.ProductName : snapMeta?.ProductName ?? string.Empty; + var platform = !string.IsNullOrWhiteSpace(dbMeta.Platform) ? dbMeta.Platform : snapMeta?.Platform ?? string.Empty; + var unityVersion = dbMeta.UnityVersion ?? string.Empty; + if (string.IsNullOrWhiteSpace(unityVersion) || unityVersion.StartsWith("format:", StringComparison.OrdinalIgnoreCase)) + unityVersion = snapMeta?.UnityVersion ?? unityVersion; + + var snapFormat = dbMeta.SnapFormatVersion != 0 ? dbMeta.SnapFormatVersion : snapMeta?.SnapFormatVersion ?? 0; + if (snapFormat == 0 && unityVersion.StartsWith("format:", StringComparison.OrdinalIgnoreCase) + && uint.TryParse(unityVersion["format:".Length..], NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)) + { + snapFormat = parsed; + } + + var sortTimestamp = dbMeta.RecordDateUtc + ?? snapMeta?.RecordDateUtc + ?? ParseCaptureDateFromFileName(fileName); + + return new EnrichedSnapshotMetadata + { + SessionGuid = sessionGuid, + ProductName = productName, + Platform = platform, + UnityVersionDisplay = unityVersion, + SnapFormatVersion = snapFormat, + SortTimestamp = sortTimestamp, + }; + } + + private static DateTime ParseCaptureDateFromFileName(string fileName) + { + var match = System.Text.RegularExpressions.Regex.Match( + fileName, + @"_(?\d{4}-\d{2}-\d{2})_(?