// lib/storage/sqlite_to_sidecar_migrator.dart // // Phase 5 of the file-based storage plan (docs/plans/2026-06-24-file-based- // storage.md §B): a ONE-TIME, ADDITIVE, IDEMPOTENT migration of legacy SQLite // data into vault sidecars. A user upgrading from the SQLite era has existing // documents/notes/ink/scratchpads/bookmarks that the new editors no longer // write to; this lifts that data into `.badnote.json` sidecars so nothing // is lost. // // Guarantees (§B / acceptance): // * NEVER loses data. The legacy DB is renamed to `*.premigration`, never // deleted, BEFORE the migration flag is flipped (so a failed run loses // nothing and the caller can retry). // * Additive: only WRITES sidecars + COPIES source files into the vault. Reads // the legacy DB read-only. // * Idempotent / re-runnable: a target sidecar that already exists is skipped, // so a half-finished run resumes on relaunch and a completed run is a no-op. // * Missing-source-graceful: if a legacy document's source file is gone, its // annotations (the precious part) are still migrated into a notebook folder; // the absence is recorded in [MigrationReport.missingSources], not fatal. // // Mapping (table → sidecar field): // documents → a notebook folder + `.badnote.json` // ink (host page) → sidecar.strokes[pageIndex] (EditorStroke JSON) // bookmarks → sidecar.bookmarks (Bookmark JSON) // scratch_links → sidecar.scratchLinks[].link (ScratchLink JSON) // scratchpads → sidecar.scratchLinks[].scratchpad (InkStroke JSON, abs px) // annotations → sidecar.legacyAnnotations (raw blob, never dropped) // highlights → none in legacy data (always empty) // notes + strokes → a standalone notebook folder + `notebook.badnote.json` // (strokes normalized onto page 0 as EditorStroke, exactly // as the runtime note editor persists them) import 'dart:io'; import 'package:path/path.dart' as p; import 'package:sqflite_common_ffi/sqflite_ffi.dart'; import '../editor/engine/stroke_model.dart'; import '../editor/notebook/ink_stroke_adapter.dart'; import '../models/document.dart' as doc; import '../models/ink_stroke.dart'; import '../services/database_service.dart'; import '../services/vault_service.dart'; import 'badnote_sidecar.dart'; import 'sidecar_store.dart'; /// Outcome of one [SqliteToSidecarMigrator.run] call. Carries counts + the /// notebook/source paths touched so callers (and tests) can assert coverage and /// surface a brief summary. class MigrationReport { MigrationReport({ this.documentsMigrated = 0, this.documentsSkipped = 0, this.notesMigrated = 0, this.notesSkipped = 0, List? missingSources, this.legacyDbFound = false, this.legacyDbPreservedPath, }) : missingSources = missingSources ?? []; /// File-backed documents written as new sidecars this run. int documentsMigrated; /// File-backed documents skipped because their sidecar already existed. int documentsSkipped; /// Standalone notes written as new `notebook.badnote.json` this run. int notesMigrated; /// Standalone notes skipped because their sidecar already existed. int notesSkipped; /// Filenames of legacy documents whose source file no longer existed on disk /// (annotations were still migrated into a notebook folder without a file). final List missingSources; /// Whether a legacy DB file was actually found and opened. bool legacyDbFound; /// Where the legacy DB ended up (its `*.premigration` path), or null if there /// was no legacy DB to preserve. String? legacyDbPreservedPath; bool get didAnything => documentsMigrated > 0 || notesMigrated > 0; @override String toString() => 'MigrationReport(documentsMigrated: $documentsMigrated, ' 'documentsSkipped: $documentsSkipped, notesMigrated: $notesMigrated, ' 'notesSkipped: $notesSkipped, missingSources: ${missingSources.length}, ' 'legacyDbFound: $legacyDbFound)'; } /// One-time SQLite → sidecar migrator. Construct with the target [VaultService] /// and (optionally) an explicit legacy DB path for tests; call [run] once. class SqliteToSidecarMigrator { SqliteToSidecarMigrator(this._vault, {String? legacyDbPath}) : _legacyDbPathOverride = legacyDbPath; final VaultService _vault; final String? _legacyDbPathOverride; /// Suffix the legacy DB is renamed to so it is preserved (never destroyed). static const String premigrationSuffix = '.premigration'; /// Logical page a legacy free-ink note was drawn on (its `InkStroke`s are in /// absolute pixels on this rect). Migrated strokes are normalized against it, /// exactly as the runtime note editor does on load. static const _noteLogicalPage = kNoteLogicalPage; /// Run the migration. Safe to call when there is nothing to migrate (fresh /// install → no legacy DB → no-op). Returns a [MigrationReport]. /// /// Throws [StateError] if the vault root is not valid (callers must gate on a /// valid vault first). Future run() async { if (!await _vault.vaultRootValid()) { throw StateError('Cannot migrate: vault root is not valid.'); } final report = MigrationReport(); final legacyPath = _legacyDbPathOverride ?? await DatabaseService.legacyDbPath(); final legacyFile = File(legacyPath); if (!await legacyFile.exists()) { // Fresh install (or already migrated + renamed): nothing to do. return report; } report.legacyDbFound = true; // sqflite ffi must be initialised before opening (the migrator may run on // desktop before any DatabaseService.getInstance call). if (Platform.isLinux || Platform.isWindows || Platform.isMacOS) { sqfliteFfiInit(); } final db = await databaseFactoryFfi.openDatabase( legacyPath, options: OpenDatabaseOptions(readOnly: true, singleInstance: false), ); try { await _migrateDocuments(db, report); await _migrateNotes(db, report); } finally { await db.close(); } // Preserve the legacy DB as `*.premigration` (never delete). Done AFTER a // successful pass so a crash mid-migration leaves the original in place for // a clean retry. Idempotent: if already renamed on a prior run, skip. final preserved = File('$legacyPath$premigrationSuffix'); if (!await preserved.exists()) { await legacyFile.rename(preserved.path); } report.legacyDbPreservedPath = preserved.path; return report; } Future _migrateDocuments(Database db, MigrationReport report) async { final documents = await DatabaseService.rawAllDocuments(db); // Robust idempotency: a set of legacy ids already migrated, recovered by // scanning every existing sidecar's `legacyId`. Re-running recognizes // already-migrated items even if folder names collided. final migratedIds = await _migratedLegacyIds(); for (final document in documents) { if (migratedIds.contains(document.id)) { report.documentsSkipped++; continue; } final sourceExists = await File(document.filePath).exists(); // Locate/create the notebook folder. Reuse VaultService.createNotebook // (folder + file copy) when the source exists; otherwise make an // annotations-only folder so the precious ink is never lost. final String vaultSourcePath; if (sourceExists) { vaultSourcePath = await _vault.createNotebook(document.filePath); } else { report.missingSources.add(document.filename); vaultSourcePath = await _ensureNotebookForMissingSource( document.filename, ); } final sidecarFile = File('$vaultSourcePath$kVaultSidecarSuffix'); final sidecar = await _buildDocumentSidecar(db, document, vaultSourcePath); await SidecarStore.writeAtomic(sidecarFile, sidecar); migratedIds.add(document.id); report.documentsMigrated++; } } Future _buildDocumentSidecar( Database db, doc.Document document, String vaultSourcePath, ) async { final documentId = document.id; final strokes = await DatabaseService.rawStrokesByPage(db, documentId); final bookmarks = await DatabaseService.rawBookmarks(db, documentId); final legacyAnnotations = await DatabaseService.rawLegacyAnnotations(db, documentId); final links = await DatabaseService.rawScratchLinks(db, documentId); final scratchLinks = []; for (final link in links) { // The scratchpad row is keyed by the ANCHOR id (see saveScratchpad). final pad = await DatabaseService.rawScratchpad(db, link.id); scratchLinks.add(SidecarScratchLink( link: link, scratchpad: SidecarScratchpad(strokes: pad), )); } return BadnoteSidecar( sourceFile: p.basename(vaultSourcePath), docType: document.docType, pageCount: document.pageCount, rotation: document.rotation, createdAt: document.createdAt, updatedAt: document.updatedAt, strokes: strokes, bookmarks: bookmarks, scratchLinks: scratchLinks, legacyAnnotations: legacyAnnotations, legacyId: documentId, ); } Future _migrateNotes(Database db, MigrationReport report) async { final notes = await DatabaseService.rawAllNotes(db); final migratedIds = await _migratedLegacyIds(); for (final note in notes) { // Idempotent: recognize an already-migrated note by its legacy id stored // in some existing `notebook.badnote.json` (folder name may have collided // / been de-duped, so a path guess is unreliable — the id is canonical). if (migratedIds.contains(note.id)) { report.notesSkipped++; continue; } // Create the standalone notebook folder + initial sidecar (title), then // overwrite the sidecar with the migrated strokes on page 0. final notePath = await _vault.createEmptyNotebook(note.title); // Legacy note strokes are InkStroke in ABSOLUTE px on the note's logical // page. Normalize them the same way the runtime note editor does on load // (penStrokeFromInk → EditorStroke.fromPenStroke) so the migrated note // renders identically. final editorStrokes = [ for (final InkStroke s in note.strokes) if (_toEditor(s) case final EditorStroke es) es, ]; final now = DateTime.now().toUtc(); final sidecar = BadnoteSidecar( docType: 'notebook', title: note.title.trim().isEmpty ? null : note.title.trim(), createdAt: note.createdAt, updatedAt: note.updatedAt.isAfter(now) ? now : note.updatedAt, strokes: editorStrokes.isEmpty ? null : {0: editorStrokes}, legacyId: note.id, ); await SidecarStore.writeAtomic( File('$notePath$kVaultSidecarSuffix'), sidecar, ); migratedIds.add(note.id); report.notesMigrated++; } } /// Convert a legacy note [InkStroke] (absolute px) to a normalized /// [EditorStroke] via the exact runtime chain. Returns null for non-freehand /// strokes (shapes/text), which the pen canvas cannot represent. EditorStroke? _toEditor(InkStroke s) { final pen = penStrokeFromInk(s, _noteLogicalPage); if (pen == null) return null; return EditorStroke.fromPenStroke(pen, id: s.id); } /// Scan every notebook sidecar already in the vault and collect the `legacyId` /// values, so the migration can recognize already-migrated documents/notes on /// a re-run regardless of any folder-name de-duplication. A fresh vault yields /// an empty set. Future> _migratedLegacyIds() async { final root = _vault.vaultRoot; final ids = {}; if (root == null || root.isEmpty) return ids; final dir = Directory(root); if (!await dir.exists()) return ids; await for (final entity in dir.list(followLinks: false)) { if (entity is! Directory) continue; if (p.basename(entity.path).startsWith('.')) continue; await for (final file in entity.list(followLinks: false)) { if (file is! File) continue; if (!file.path.endsWith(kVaultSidecarSuffix)) continue; // Skip .bak/.tmp variants (they don't end in the suffix anyway). final sidecar = await SidecarStore.read(file); final id = sidecar?.legacyId; if (id != null) ids.add(id); } } return ids; } /// Ensure an annotations-only notebook folder exists for a legacy document /// whose source file is GONE. Returns the synthetic source path the sidecar /// keys off (`/`), so the sidecar lands at /// `/.badnote.json` — identical to the file-backed case but /// with no copied file. Idempotent on re-run. Future _ensureNotebookForMissingSource(String filename) async { final root = _vault.vaultRoot!; final baseName = _sanitize(p.basenameWithoutExtension(filename)); final folder = Directory(p.join(root, baseName.isEmpty ? 'Untitled' : baseName)); final syntheticSource = p.join(folder.path, filename); if (await File('$syntheticSource$kVaultSidecarSuffix').exists()) { return syntheticSource; // already migrated } await folder.create(recursive: true); return syntheticSource; } /// Mirror of VaultService._sanitizeFolderName (kept private there) so the /// missing-source notebook folder lands at the same name the source-backed /// path would have used. static String _sanitize(String name) => name .replaceAll(RegExp(r'[\\/:*?"<>|\x00-\x1f]'), ' ') .replaceAll(RegExp(r'\s+'), ' ') .trim() .replaceAll(RegExp(r'[. ]+$'), ''); }