diff --git a/lib/main.dart b/lib/main.dart index ec3a9ec..567807d 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -13,6 +13,7 @@ import 'screens/home_screen.dart'; import 'screens/vault_setup_screen.dart'; import 'services/database_service.dart'; import 'services/vault_service.dart'; +import 'storage/sqlite_to_sidecar_migrator.dart'; Future main() async { // Kind-aware binding (extends WidgetsFlutterBinding) must be the active @@ -98,6 +99,7 @@ class _VaultGateState extends State { bool _valid = false; bool _hadStoredPath = false; bool _loading = true; + bool _migrating = false; @override void initState() { @@ -116,6 +118,27 @@ class _VaultGateState extends State { _hadStoredPath = (vault.vaultRoot?.isNotEmpty ?? false); _loading = false; }); + if (valid) await _maybeMigrate(vault); + } + + /// Run the one-time SQLite→sidecar migration ONCE per vault (Phase 5, §B). + /// Gated on [VaultService.vaultMigrationDone]; a fresh install (no legacy DB) + /// is a fast no-op. The live DB is first reopened at the vault cache location + /// so post-migration reads hit the new index, never the renamed legacy file. + Future _maybeMigrate(VaultService vault) async { + // Move the live cache DB to the vault location now that the root is valid. + await DatabaseService.reopen(); + if (vault.vaultMigrationDone) return; + if (mounted) setState(() => _migrating = true); + try { + await SqliteToSidecarMigrator(vault).run(); + await vault.setVaultMigrationDone(); + } catch (_) { + // A failed migration leaves the legacy DB intact (it is only renamed to + // `.premigration` after a successful pass) and the flag unset, so the + // next launch retries. Never block the user from reaching the app. + } + if (mounted) setState(() => _migrating = false); } @override @@ -125,11 +148,28 @@ class _VaultGateState extends State { body: Center(child: CircularProgressIndicator()), ); } + if (_migrating) { + return const Scaffold( + body: Center( + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + CircularProgressIndicator(), + SizedBox(height: 16), + Text('Migrating your notebooks…'), + ], + ), + ), + ); + } if (_valid) return const HomeScreen(); return VaultSetupScreen( vaultService: _vault!, missing: _hadStoredPath, - onVaultReady: () => setState(() => _valid = true), + onVaultReady: () async { + if (_vault != null) await _maybeMigrate(_vault!); + if (mounted) setState(() => _valid = true); + }, ); } } diff --git a/lib/services/database_service.dart b/lib/services/database_service.dart index a1778bc..f1e0b41 100644 --- a/lib/services/database_service.dart +++ b/lib/services/database_service.dart @@ -5,10 +5,12 @@ import 'dart:ui' show Offset, Size; import 'package:flutter/foundation.dart' show visibleForTesting; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; +import 'package:shared_preferences/shared_preferences.dart'; import 'package:sqflite_common_ffi/sqflite_ffi.dart'; import 'package:uuid/uuid.dart'; import '../editor/board/board.dart'; +import '../editor/engine/stroke_model.dart'; import '../models/bookmark.dart'; import '../models/document.dart' as doc; import '../models/ink_point.dart'; @@ -17,6 +19,7 @@ import '../models/note.dart'; import '../models/pen_tool.dart'; import '../models/pointer_device_kind.dart'; import '../models/scratch_link.dart'; +import 'vault_service.dart'; class DatabaseService { static DatabaseService? _instance; @@ -46,14 +49,30 @@ class DatabaseService { Database get database => _database; + /// Re-resolve the DB location and reopen the singleton there. Called once the + /// vault root becomes valid at startup so the live database moves from the + /// legacy app-documents `badnote.db` to the vault cache + /// `/.badnote/index.sqlite` (§A.1). No-op-safe: if the resolved path is + /// unchanged it simply reopens the same file. Closes the previous handle. + static Future reopen() async { + final existing = _instance; + if (existing != null) { + await existing._database.close(); + _instance = null; + } + return getInstance(); + } + Future _initialize() async { if (Platform.isLinux || Platform.isWindows || Platform.isMacOS) { sqfliteFfiInit(); databaseFactory = databaseFactoryFfi; } - final dir = await getApplicationDocumentsDirectory(); - final dbPath = p.join(dir.path, 'badnote.db'); + final dbPath = await _resolveDbPath(); + // Ensure the parent dir exists (the vault's hidden `.badnote/` cache dir is + // not guaranteed to exist yet on first run). + await Directory(p.dirname(dbPath)).create(recursive: true); _database = await openDatabase( dbPath, @@ -63,6 +82,36 @@ class DatabaseService { ); } + /// The application-documents path of the LEGACY (pre-vault) database. This is + /// the location [DatabaseService] used before the file-based re-architecture; + /// the one-time migrator reads from here, then renames it to `.premigration`. + static Future legacyDbPath() async { + final dir = await getApplicationDocumentsDirectory(); + return p.join(dir.path, 'badnote.db'); + } + + /// Resolve where the live database should live. When a valid vault root is + /// set, the DB is the vault's rebuildable cache/index at + /// `/.badnote/index.sqlite` (§A.1). Otherwise (no vault yet — e.g. a + /// fresh first run before the gate, or tests) fall back to the legacy + /// app-documents `badnote.db` so the app still works. + Future _resolveDbPath() async { + String? root; + try { + final prefs = await SharedPreferences.getInstance(); + root = prefs.getString(VaultService.vaultRootKey); + } catch (_) { + // SharedPreferences may be unavailable (e.g. a unit test that mocks only + // the path provider). Fall back to the legacy app-documents location so + // the DB still opens — it is never the source of truth anyway. + root = null; + } + if (root != null && root.isNotEmpty && await Directory(root).exists()) { + return p.join(root, '.badnote', 'index.sqlite'); + } + return legacyDbPath(); + } + Future _onCreate(Database db, int version) async { // Core tables (original v1) await db.execute(''' @@ -1083,4 +1132,176 @@ class DatabaseService { await txn.delete('scratchpads', where: 'document_id = ?', whereArgs: [id]); }); } + + // ── RAW legacy reads (one-time SQLite→sidecar migration, Phase 5) ─────────── + // + // These operate on an arbitrary [Database] handle (the LEGACY db the migrator + // opens directly), NOT the live [_database] cache, so the migrator can read + // pre-migration data without touching the relocated index. They reuse this + // class's row-parsers so the JSON shapes stay identical to the live reads. + + /// All `documents` rows from [db], oldest first (stable migration order). + static Future> rawAllDocuments(Database db) async { + if (!await _tableExists(db, 'documents')) return const []; + final rows = await db.query('documents', orderBy: 'created_at ASC'); + final dummy = DatabaseService._(); + return rows.map(dummy._documentFromRow).toList(); + } + + /// All `notes` rows (with their `strokes`) from [db], oldest first. + static Future> rawAllNotes(Database db) async { + if (!await _tableExists(db, 'notes')) return const []; + final rows = await db.query('notes', orderBy: 'created_at ASC'); + final dummy = DatabaseService._(); + final notes = []; + for (final row in rows) { + final strokeRows = await db.query( + 'strokes', + where: 'note_id = ?', + whereArgs: [row['id'] as String], + orderBy: 'created_at ASC', + ); + final strokes = strokeRows.map(dummy._strokeFromRow).toList(); + final tagsJson = jsonDecode(row['tags'] as String) as List; + notes.add(Note( + id: row['id'] as String, + title: row['title'] as String, + strokes: strokes, + createdAt: DateTime.parse(row['created_at'] as String), + updatedAt: DateTime.parse(row['updated_at'] as String), + tags: tagsJson.cast(), + )); + } + return notes; + } + + /// Committed editor strokes for [documentId] from [db], grouped by 0-based + /// page index. Parses the `ink.host_id = "doc::page:"` scheme + /// (see [EditorRepository.loadDocument]) and decodes each `stroke_json` + /// straight into an [EditorStroke]. Returns `{}` when there is no `ink` table + /// or no rows. + static Future>> rawStrokesByPage( + Database db, + String documentId, + ) async { + if (!await _tableExists(db, 'ink')) return >{}; + final rows = await db.query( + 'ink', + where: 'host_kind = ? AND host_id LIKE ?', + whereArgs: ['page', 'doc:$documentId:page:%'], + orderBy: 'host_id ASC, ordinal ASC', + ); + final out = >{}; + for (final row in rows) { + final hostId = row['host_id'] as String; + final pageIndex = _pageIndexFromHostId(hostId); + if (pageIndex == null) continue; + final json = + jsonDecode(row['stroke_json'] as String) as Map; + out.putIfAbsent(pageIndex, () => []).add(EditorStroke.fromJson(json)); + } + return out; + } + + /// Bookmarks for [documentId] from [db] (empty when no `bookmarks` table). + static Future> rawBookmarks( + Database db, + String documentId, + ) async { + if (!await _tableExists(db, 'bookmarks')) return const []; + final rows = await db.query( + 'bookmarks', + where: 'document_id = ?', + whereArgs: [documentId], + orderBy: 'page_number ASC', + ); + final dummy = DatabaseService._(); + return rows.map(dummy._bookmarkFromRow).toList(); + } + + /// Scratch-link anchors for [documentId] from [db] (empty when no table). + static Future> rawScratchLinks( + Database db, + String documentId, + ) async { + if (!await _tableExists(db, 'scratch_links')) return const []; + final rows = await db.query( + 'scratch_links', + where: 'document_id = ?', + whereArgs: [documentId], + orderBy: 'created_at ASC', + ); + return rows + .map( + (row) => ScratchLink( + id: row['id'] as String, + documentId: row['document_id'] as String, + pageIndex: row['page_index'] as int, + nx: (row['nx'] as num).toDouble(), + ny: (row['ny'] as num).toDouble(), + ), + ) + .toList(); + } + + /// Scratchpad strokes stored under [key] (an anchor id) from [db]. Empty when + /// there is no `scratchpads` table or no row. + static Future> rawScratchpad( + Database db, + String key, + ) async { + if (!await _tableExists(db, 'scratchpads')) return const []; + final rows = await db.query( + 'scratchpads', + where: 'document_id = ?', + whereArgs: [key], + ); + if (rows.isEmpty) return const []; + final json = rows.first['strokes_json'] as String; + if (json.isEmpty || json == '[]') return const []; + final list = jsonDecode(json) as List; + return list + .map((s) => InkStroke.fromJson(s as Map)) + .toList(); + } + + /// Raw legacy per-page `annotation_json` blobs for [documentId] from [db], + /// keyed by page number. These belong to the DEAD pre-editor annotation path + /// (§1 `annotations` table); the migrator copies them verbatim into the + /// sidecar's `legacyAnnotations` so nothing is silently dropped. + static Future> rawLegacyAnnotations( + Database db, + String documentId, + ) async { + if (!await _tableExists(db, 'annotations')) return {}; + final rows = await db.query( + 'annotations', + where: 'document_id = ?', + whereArgs: [documentId], + orderBy: 'page_number ASC', + ); + final out = {}; + for (final row in rows) { + out[row['page_number'] as int] = row['annotation_json'] as String; + } + return out; + } + + /// Parse the 0-based page index out of an `ink.host_id` of the form + /// `doc::page:`. Returns null on an unexpected shape. + static int? _pageIndexFromHostId(String hostId) { + final i = hostId.lastIndexOf(':page:'); + if (i == -1) return null; + return int.tryParse(hostId.substring(i + ':page:'.length)); + } + + /// True iff [name] is an existing table in [db]. Lets the raw readers tolerate + /// a legacy DB that predates a given table (older schema versions). + static Future _tableExists(Database db, String name) async { + final rows = await db.rawQuery( + "SELECT name FROM sqlite_master WHERE type='table' AND name=?", + [name], + ); + return rows.isNotEmpty; + } } diff --git a/lib/services/vault_service.dart b/lib/services/vault_service.dart index 0684a97..42bf1be 100644 --- a/lib/services/vault_service.dart +++ b/lib/services/vault_service.dart @@ -92,6 +92,10 @@ class VaultService { /// SharedPreferences key under which the vault root path is stored. static const String vaultRootKey = 'vaultRoot'; + /// SharedPreferences key gating the one-time SQLite→sidecar migration + /// (Phase 5). Set true once the migration completes so it never re-runs. + static const String vaultMigrationDoneKey = 'vaultMigrationDone'; + final SharedPreferences _prefs; VaultService._(this._prefs); @@ -132,6 +136,15 @@ class VaultService { await _prefs.remove(vaultRootKey); } + /// True once the one-time SQLite→sidecar migration (Phase 5) has completed. + /// When false, startup runs the migrator before opening the home screen. + bool get vaultMigrationDone => _prefs.getBool(vaultMigrationDoneKey) ?? false; + + /// Mark the one-time SQLite→sidecar migration as done so it never re-runs. + Future setVaultMigrationDone() async { + await _prefs.setBool(vaultMigrationDoneKey, true); + } + /// True iff a vault root is set AND that directory currently exists. /// /// Returns false when no path is stored or when the stored path no longer diff --git a/lib/storage/badnote_sidecar.dart b/lib/storage/badnote_sidecar.dart index 06ef8ff..1c83bb4 100644 --- a/lib/storage/badnote_sidecar.dart +++ b/lib/storage/badnote_sidecar.dart @@ -221,10 +221,13 @@ class BadnoteSidecar { Map>? highlights, List? bookmarks, List? scratchLinks, + Map? legacyAnnotations, + this.legacyId, }) : strokes = strokes ?? >{}, highlights = highlights ?? >{}, bookmarks = bookmarks ?? [], - scratchLinks = scratchLinks ?? []; + scratchLinks = scratchLinks ?? [], + legacyAnnotations = legacyAnnotations ?? {}; /// Schema version (`badnoteSidecarVersion`). final int version; @@ -253,6 +256,19 @@ class BadnoteSidecar { final List bookmarks; final List scratchLinks; + /// Raw legacy per-page `annotation_json` blobs preserved verbatim from the + /// DEAD pre-editor `annotations` SQLite table (keyed by page number). Populated + /// only by the one-time SQLite→sidecar migration so no legacy data is silently + /// dropped; the live editor ignores it. Empty for all freshly authored + /// sidecars. + final Map legacyAnnotations; + + /// The legacy SQLite row id this sidecar was migrated from (a `documents.id` + /// or `notes.id`). Set ONLY by the one-time migration; it makes the migration + /// idempotent (a re-run recognizes an already-migrated item by this id even if + /// its folder name collided). Null for all freshly authored sidecars. + final String? legacyId; + Map toJson() => { 'badnoteSidecarVersion': version, if (sourceFile != null) 'sourceFile': sourceFile, @@ -274,6 +290,12 @@ class BadnoteSidecar { }, 'bookmarks': bookmarks.map((b) => b.toJson()).toList(), 'scratchLinks': scratchLinks.map((s) => s.toJson()).toList(), + if (legacyAnnotations.isNotEmpty) + 'legacyAnnotations': { + for (final entry in legacyAnnotations.entries) + entry.key.toString(): entry.value, + }, + if (legacyId != null) 'legacyId': legacyId, }; factory BadnoteSidecar.fromJson(Map json) { @@ -316,6 +338,18 @@ class BadnoteSidecar { scratchLinks: ((json['scratchLinks'] as List?) ?? const []) .map((e) => SidecarScratchLink.fromJson(e as Map)) .toList(), + legacyAnnotations: () { + final raw = json['legacyAnnotations']; + final out = {}; + if (raw is Map) { + raw.forEach((key, value) { + final page = int.tryParse(key.toString()); + if (page != null && value is String) out[page] = value; + }); + } + return out; + }(), + legacyId: json['legacyId'] as String?, ); } } diff --git a/lib/storage/sqlite_to_sidecar_migrator.dart b/lib/storage/sqlite_to_sidecar_migrator.dart new file mode 100644 index 0000000..a7b0446 --- /dev/null +++ b/lib/storage/sqlite_to_sidecar_migrator.dart @@ -0,0 +1,335 @@ +// lib/storage/sqlite_to_sidecar_migrator.dart +// +// Phase 5 of the file-based storage plan (docs/plans/2026-06-24-file-based- +// storage.md §B): a ONE-TIME, ADDITIVE, IDEMPOTENT migration of legacy SQLite +// data into vault sidecars. A user upgrading from the SQLite era has existing +// documents/notes/ink/scratchpads/bookmarks that the new editors no longer +// write to; this lifts that data into `.badnote.json` sidecars so nothing +// is lost. +// +// Guarantees (§B / acceptance): +// * NEVER loses data. The legacy DB is renamed to `*.premigration`, never +// deleted, BEFORE the migration flag is flipped (so a failed run loses +// nothing and the caller can retry). +// * Additive: only WRITES sidecars + COPIES source files into the vault. Reads +// the legacy DB read-only. +// * Idempotent / re-runnable: a target sidecar that already exists is skipped, +// so a half-finished run resumes on relaunch and a completed run is a no-op. +// * Missing-source-graceful: if a legacy document's source file is gone, its +// annotations (the precious part) are still migrated into a notebook folder; +// the absence is recorded in [MigrationReport.missingSources], not fatal. +// +// Mapping (table → sidecar field): +// documents → a notebook folder + `.badnote.json` +// ink (host page) → sidecar.strokes[pageIndex] (EditorStroke JSON) +// bookmarks → sidecar.bookmarks (Bookmark JSON) +// scratch_links → sidecar.scratchLinks[].link (ScratchLink JSON) +// scratchpads → sidecar.scratchLinks[].scratchpad (InkStroke JSON, abs px) +// annotations → sidecar.legacyAnnotations (raw blob, never dropped) +// highlights → none in legacy data (always empty) +// notes + strokes → a standalone notebook folder + `notebook.badnote.json` +// (strokes normalized onto page 0 as EditorStroke, exactly +// as the runtime note editor persists them) + +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import '../editor/engine/stroke_model.dart'; +import '../editor/notebook/ink_stroke_adapter.dart'; +import '../models/document.dart' as doc; +import '../models/ink_stroke.dart'; +import '../services/database_service.dart'; +import '../services/vault_service.dart'; +import 'badnote_sidecar.dart'; +import 'sidecar_store.dart'; + +/// Outcome of one [SqliteToSidecarMigrator.run] call. Carries counts + the +/// notebook/source paths touched so callers (and tests) can assert coverage and +/// surface a brief summary. +class MigrationReport { + MigrationReport({ + this.documentsMigrated = 0, + this.documentsSkipped = 0, + this.notesMigrated = 0, + this.notesSkipped = 0, + List? missingSources, + this.legacyDbFound = false, + this.legacyDbPreservedPath, + }) : missingSources = missingSources ?? []; + + /// File-backed documents written as new sidecars this run. + int documentsMigrated; + + /// File-backed documents skipped because their sidecar already existed. + int documentsSkipped; + + /// Standalone notes written as new `notebook.badnote.json` this run. + int notesMigrated; + + /// Standalone notes skipped because their sidecar already existed. + int notesSkipped; + + /// Filenames of legacy documents whose source file no longer existed on disk + /// (annotations were still migrated into a notebook folder without a file). + final List missingSources; + + /// Whether a legacy DB file was actually found and opened. + bool legacyDbFound; + + /// Where the legacy DB ended up (its `*.premigration` path), or null if there + /// was no legacy DB to preserve. + String? legacyDbPreservedPath; + + bool get didAnything => + documentsMigrated > 0 || notesMigrated > 0; + + @override + String toString() => 'MigrationReport(documentsMigrated: $documentsMigrated, ' + 'documentsSkipped: $documentsSkipped, notesMigrated: $notesMigrated, ' + 'notesSkipped: $notesSkipped, missingSources: ${missingSources.length}, ' + 'legacyDbFound: $legacyDbFound)'; +} + +/// One-time SQLite → sidecar migrator. Construct with the target [VaultService] +/// and (optionally) an explicit legacy DB path for tests; call [run] once. +class SqliteToSidecarMigrator { + SqliteToSidecarMigrator(this._vault, {String? legacyDbPath}) + : _legacyDbPathOverride = legacyDbPath; + + final VaultService _vault; + final String? _legacyDbPathOverride; + + /// Suffix the legacy DB is renamed to so it is preserved (never destroyed). + static const String premigrationSuffix = '.premigration'; + + /// Logical page a legacy free-ink note was drawn on (its `InkStroke`s are in + /// absolute pixels on this rect). Migrated strokes are normalized against it, + /// exactly as the runtime note editor does on load. + static const _noteLogicalPage = kNoteLogicalPage; + + /// Run the migration. Safe to call when there is nothing to migrate (fresh + /// install → no legacy DB → no-op). Returns a [MigrationReport]. + /// + /// Throws [StateError] if the vault root is not valid (callers must gate on a + /// valid vault first). + Future run() async { + if (!await _vault.vaultRootValid()) { + throw StateError('Cannot migrate: vault root is not valid.'); + } + + final report = MigrationReport(); + + final legacyPath = _legacyDbPathOverride ?? await DatabaseService.legacyDbPath(); + final legacyFile = File(legacyPath); + if (!await legacyFile.exists()) { + // Fresh install (or already migrated + renamed): nothing to do. + return report; + } + report.legacyDbFound = true; + + // sqflite ffi must be initialised before opening (the migrator may run on + // desktop before any DatabaseService.getInstance call). + if (Platform.isLinux || Platform.isWindows || Platform.isMacOS) { + sqfliteFfiInit(); + } + final db = await databaseFactoryFfi.openDatabase( + legacyPath, + options: OpenDatabaseOptions(readOnly: true, singleInstance: false), + ); + try { + await _migrateDocuments(db, report); + await _migrateNotes(db, report); + } finally { + await db.close(); + } + + // Preserve the legacy DB as `*.premigration` (never delete). Done AFTER a + // successful pass so a crash mid-migration leaves the original in place for + // a clean retry. Idempotent: if already renamed on a prior run, skip. + final preserved = File('$legacyPath$premigrationSuffix'); + if (!await preserved.exists()) { + await legacyFile.rename(preserved.path); + } + report.legacyDbPreservedPath = preserved.path; + + return report; + } + + Future _migrateDocuments(Database db, MigrationReport report) async { + final documents = await DatabaseService.rawAllDocuments(db); + // Robust idempotency: a set of legacy ids already migrated, recovered by + // scanning every existing sidecar's `legacyId`. Re-running recognizes + // already-migrated items even if folder names collided. + final migratedIds = await _migratedLegacyIds(); + for (final document in documents) { + if (migratedIds.contains(document.id)) { + report.documentsSkipped++; + continue; + } + + final sourceExists = await File(document.filePath).exists(); + + // Locate/create the notebook folder. Reuse VaultService.createNotebook + // (folder + file copy) when the source exists; otherwise make an + // annotations-only folder so the precious ink is never lost. + final String vaultSourcePath; + if (sourceExists) { + vaultSourcePath = await _vault.createNotebook(document.filePath); + } else { + report.missingSources.add(document.filename); + vaultSourcePath = await _ensureNotebookForMissingSource( + document.filename, + ); + } + + final sidecarFile = File('$vaultSourcePath$kVaultSidecarSuffix'); + final sidecar = await _buildDocumentSidecar(db, document, vaultSourcePath); + await SidecarStore.writeAtomic(sidecarFile, sidecar); + migratedIds.add(document.id); + report.documentsMigrated++; + } + } + + Future _buildDocumentSidecar( + Database db, + doc.Document document, + String vaultSourcePath, + ) async { + final documentId = document.id; + final strokes = await DatabaseService.rawStrokesByPage(db, documentId); + final bookmarks = await DatabaseService.rawBookmarks(db, documentId); + final legacyAnnotations = + await DatabaseService.rawLegacyAnnotations(db, documentId); + + final links = await DatabaseService.rawScratchLinks(db, documentId); + final scratchLinks = []; + for (final link in links) { + // The scratchpad row is keyed by the ANCHOR id (see saveScratchpad). + final pad = await DatabaseService.rawScratchpad(db, link.id); + scratchLinks.add(SidecarScratchLink( + link: link, + scratchpad: SidecarScratchpad(strokes: pad), + )); + } + + return BadnoteSidecar( + sourceFile: p.basename(vaultSourcePath), + docType: document.docType, + pageCount: document.pageCount, + rotation: document.rotation, + createdAt: document.createdAt, + updatedAt: document.updatedAt, + strokes: strokes, + bookmarks: bookmarks, + scratchLinks: scratchLinks, + legacyAnnotations: legacyAnnotations, + legacyId: documentId, + ); + } + + Future _migrateNotes(Database db, MigrationReport report) async { + final notes = await DatabaseService.rawAllNotes(db); + final migratedIds = await _migratedLegacyIds(); + for (final note in notes) { + // Idempotent: recognize an already-migrated note by its legacy id stored + // in some existing `notebook.badnote.json` (folder name may have collided + // / been de-duped, so a path guess is unreliable — the id is canonical). + if (migratedIds.contains(note.id)) { + report.notesSkipped++; + continue; + } + + // Create the standalone notebook folder + initial sidecar (title), then + // overwrite the sidecar with the migrated strokes on page 0. + final notePath = await _vault.createEmptyNotebook(note.title); + + // Legacy note strokes are InkStroke in ABSOLUTE px on the note's logical + // page. Normalize them the same way the runtime note editor does on load + // (penStrokeFromInk → EditorStroke.fromPenStroke) so the migrated note + // renders identically. + final editorStrokes = [ + for (final InkStroke s in note.strokes) + if (_toEditor(s) case final EditorStroke es) es, + ]; + + final now = DateTime.now().toUtc(); + final sidecar = BadnoteSidecar( + docType: 'notebook', + title: note.title.trim().isEmpty ? null : note.title.trim(), + createdAt: note.createdAt, + updatedAt: note.updatedAt.isAfter(now) ? now : note.updatedAt, + strokes: editorStrokes.isEmpty ? null : {0: editorStrokes}, + legacyId: note.id, + ); + await SidecarStore.writeAtomic( + File('$notePath$kVaultSidecarSuffix'), + sidecar, + ); + migratedIds.add(note.id); + report.notesMigrated++; + } + } + + /// Convert a legacy note [InkStroke] (absolute px) to a normalized + /// [EditorStroke] via the exact runtime chain. Returns null for non-freehand + /// strokes (shapes/text), which the pen canvas cannot represent. + EditorStroke? _toEditor(InkStroke s) { + final pen = penStrokeFromInk(s, _noteLogicalPage); + if (pen == null) return null; + return EditorStroke.fromPenStroke(pen, id: s.id); + } + + /// Scan every notebook sidecar already in the vault and collect the `legacyId` + /// values, so the migration can recognize already-migrated documents/notes on + /// a re-run regardless of any folder-name de-duplication. A fresh vault yields + /// an empty set. + Future> _migratedLegacyIds() async { + final root = _vault.vaultRoot; + final ids = {}; + if (root == null || root.isEmpty) return ids; + final dir = Directory(root); + if (!await dir.exists()) return ids; + + await for (final entity in dir.list(followLinks: false)) { + if (entity is! Directory) continue; + if (p.basename(entity.path).startsWith('.')) continue; + await for (final file in entity.list(followLinks: false)) { + if (file is! File) continue; + if (!file.path.endsWith(kVaultSidecarSuffix)) continue; + // Skip .bak/.tmp variants (they don't end in the suffix anyway). + final sidecar = await SidecarStore.read(file); + final id = sidecar?.legacyId; + if (id != null) ids.add(id); + } + } + return ids; + } + + /// Ensure an annotations-only notebook folder exists for a legacy document + /// whose source file is GONE. Returns the synthetic source path the sidecar + /// keys off (`/`), so the sidecar lands at + /// `/.badnote.json` — identical to the file-backed case but + /// with no copied file. Idempotent on re-run. + Future _ensureNotebookForMissingSource(String filename) async { + final root = _vault.vaultRoot!; + final baseName = _sanitize(p.basenameWithoutExtension(filename)); + final folder = Directory(p.join(root, baseName.isEmpty ? 'Untitled' : baseName)); + final syntheticSource = p.join(folder.path, filename); + if (await File('$syntheticSource$kVaultSidecarSuffix').exists()) { + return syntheticSource; // already migrated + } + await folder.create(recursive: true); + return syntheticSource; + } + + /// Mirror of VaultService._sanitizeFolderName (kept private there) so the + /// missing-source notebook folder lands at the same name the source-backed + /// path would have used. + static String _sanitize(String name) => name + .replaceAll(RegExp(r'[\\/:*?"<>|\x00-\x1f]'), ' ') + .replaceAll(RegExp(r'\s+'), ' ') + .trim() + .replaceAll(RegExp(r'[. ]+$'), ''); +} diff --git a/test/sqlite_to_sidecar_migrator_test.dart b/test/sqlite_to_sidecar_migrator_test.dart new file mode 100644 index 0000000..e68c6ed --- /dev/null +++ b/test/sqlite_to_sidecar_migrator_test.dart @@ -0,0 +1,423 @@ +// test/sqlite_to_sidecar_migrator_test.dart +// +// Golden-style proof of the one-time SQLite→sidecar migration (Phase 5, §B): +// * Seed a LEGACY sqlite DB (the pre-vault `badnote.db` schema) with a +// document (+ ink strokes on two pages, a scratch_link + its scratchpad, a +// bookmark, a legacy annotation blob) and a free-ink note (+ strokes). +// * Run the migrator against a temp vault. +// * Assert the sidecars contain the migrated data (table → field mapping). +// * Assert an idempotent re-run is a no-op (no duplicate folders/sidecars). +// * Assert the legacy DB is preserved (renamed to `*.premigration`). +// * Assert the missing-source case still migrates annotations. +// +// Uses a real ffi sqlite DB on disk (so it exercises the migrator's read path) +// and real temp dirs for the vault. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:shared_preferences/shared_preferences.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; + +import 'package:badnote/editor/notebook/ink_stroke_adapter.dart'; +import 'package:badnote/services/vault_service.dart'; +import 'package:badnote/storage/sidecar_store.dart'; +import 'package:badnote/storage/sqlite_to_sidecar_migrator.dart'; + +const _sep = '/'; // path.join uses platform sep; tests run on POSIX CI. + +/// Build the subset of the legacy v8 schema the migrator reads. +Future _openLegacyDb(String path) async { + sqfliteFfiInit(); + return databaseFactoryFfi.openDatabase( + path, + options: OpenDatabaseOptions( + version: 1, + onCreate: (db, _) async { + await db.execute(''' + CREATE TABLE documents ( + id TEXT PRIMARY KEY, filename TEXT NOT NULL, doc_type TEXT NOT NULL, + file_path TEXT NOT NULL, page_count INTEGER NOT NULL DEFAULT 0, + rotation INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL, + updated_at TEXT NOT NULL) + '''); + await db.execute(''' + CREATE TABLE annotations ( + id TEXT PRIMARY KEY, uuid TEXT NOT NULL, document_id TEXT NOT NULL, + page_number INTEGER NOT NULL, annotation_json TEXT NOT NULL, + created_at TEXT NOT NULL, updated_at TEXT NOT NULL) + '''); + await db.execute(''' + CREATE TABLE bookmarks ( + id TEXT PRIMARY KEY, document_id TEXT NOT NULL, + page_number INTEGER NOT NULL, label TEXT NOT NULL DEFAULT '', + color INTEGER NOT NULL DEFAULT 4283215696, created_at TEXT NOT NULL) + '''); + await db.execute(''' + CREATE TABLE scratchpads ( + id TEXT PRIMARY KEY, document_id TEXT UNIQUE NOT NULL, + strokes_json TEXT NOT NULL DEFAULT '[]', created_at TEXT NOT NULL, + updated_at TEXT NOT NULL) + '''); + await db.execute(''' + CREATE TABLE ink ( + id TEXT PRIMARY KEY, host_kind TEXT NOT NULL, host_id TEXT NOT NULL, + stroke_json TEXT NOT NULL, ordinal INTEGER NOT NULL, + updated_at INTEGER NOT NULL) + '''); + await db.execute(''' + CREATE TABLE scratch_links ( + id TEXT PRIMARY KEY, document_id TEXT NOT NULL, + page_index INTEGER NOT NULL, nx REAL NOT NULL, ny REAL NOT NULL, + created_at TEXT NOT NULL) + '''); + await db.execute(''' + CREATE TABLE notes ( + id TEXT PRIMARY KEY, title TEXT NOT NULL, created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, tags TEXT NOT NULL DEFAULT '[]') + '''); + await db.execute(''' + CREATE TABLE strokes ( + id TEXT PRIMARY KEY, note_id TEXT NOT NULL, tool TEXT NOT NULL, + color INTEGER NOT NULL, stroke_width REAL NOT NULL, + created_at TEXT NOT NULL, points TEXT NOT NULL) + '''); + }, + ), + ); +} + +/// A minimal EditorStroke JSON (the exact shape `ink.stroke_json` stores). +String _editorStrokeJson(String id) => jsonEncode({ + 'id': id, + 'points': [ + {'x': 0.1, 'y': 0.2, 'pressure': 0.5, 'tilt': 0.0}, + {'x': 0.3, 'y': 0.4, 'pressure': 0.6, 'tilt': 0.0}, + ], + 'tool': 'pen', + 'color': 0xFF000000, + 'width': 0.01, + }); + +/// A minimal InkStroke JSON (the exact shape `scratchpads.strokes_json` and +/// `strokes.points` produce). +Map _inkStrokeJson(String id, + {double x = 100, double y = 200}) { + return { + 'id': id, + 'points': [ + { + 'x': x, + 'y': y, + 'pressure': 0.5, + 'tilt': 0.0, + 'timestamp': 0, + 'pointerDeviceKind': 'stylus', + }, + ], + 'tool': 'pen', + 'color': 0xFF112233, + 'strokeWidth': 3.0, + 'createdAt': '2024-01-01T00:00:00.000Z', + }; +} + +void main() { + TestWidgetsFlutterBinding.ensureInitialized(); + sqfliteFfiInit(); + + late Directory tempRoot; // holds both the legacy db and the vault + late Directory vaultDir; + late Directory sourceDir; // where the "original" source file lives + late String legacyDbPath; + late VaultService vault; + + setUp(() async { + SharedPreferences.setMockInitialValues({}); + tempRoot = await Directory.systemTemp.createTemp('migrator_test_'); + vaultDir = Directory(p.join(tempRoot.path, 'vault')); + await vaultDir.create(recursive: true); + sourceDir = Directory(p.join(tempRoot.path, 'src')); + await sourceDir.create(recursive: true); + legacyDbPath = p.join(tempRoot.path, 'badnote.db'); + + final prefs = await SharedPreferences.getInstance(); + vault = VaultService.forTest(prefs); + await vault.setVaultRoot(vaultDir.path); + }); + + tearDown(() async { + if (await tempRoot.exists()) await tempRoot.delete(recursive: true); + }); + + /// Seed the legacy DB with one file-backed document and one note. Returns the + /// on-disk source file path used for the document (so callers can delete it to + /// exercise the missing-source case). + Future seedLegacy({bool createSourceFile = true}) async { + final srcPath = p.join(sourceDir.path, 'Lecture.pdf'); + if (createSourceFile) { + await File(srcPath).writeAsString('PDFDATA'); + } + + final db = await _openLegacyDb(legacyDbPath); + try { + const docId = 'doc-1'; + await db.insert('documents', { + 'id': docId, + 'filename': 'Lecture.pdf', + 'doc_type': 'pdf', + 'file_path': srcPath, + 'page_count': 5, + 'rotation': 90, + 'created_at': '2024-01-01T00:00:00.000Z', + 'updated_at': '2024-02-01T00:00:00.000Z', + }); + // Ink on page 0 and page 3. + await db.insert('ink', { + 'id': 's0', + 'host_kind': 'page', + 'host_id': 'doc:$docId:page:0', + 'stroke_json': _editorStrokeJson('s0'), + 'ordinal': 0, + 'updated_at': 0, + }); + await db.insert('ink', { + 'id': 's3', + 'host_kind': 'page', + 'host_id': 'doc:$docId:page:3', + 'stroke_json': _editorStrokeJson('s3'), + 'ordinal': 0, + 'updated_at': 0, + }); + // A bookmark. + await db.insert('bookmarks', { + 'id': 'bm-1', + 'document_id': docId, + 'page_number': 5, + 'label': 'Proof', + 'color': 4283215696, + 'created_at': '2024-01-01T00:00:00.000Z', + }); + // A scratch link + its private scratchpad (keyed by the anchor id). + await db.insert('scratch_links', { + 'id': 'anchor-1', + 'document_id': docId, + 'page_index': 7, + 'nx': 0.83, + 'ny': 0.41, + 'created_at': '2024-01-01T00:00:00.000Z', + }); + await db.insert('scratchpads', { + 'id': 'sp-1', + 'document_id': 'anchor-1', + 'strokes_json': jsonEncode([_inkStrokeJson('pad-stroke')]), + 'created_at': '2024-01-01T00:00:00.000Z', + 'updated_at': '2024-01-01T00:00:00.000Z', + }); + // A legacy per-page annotation blob (dead path → preserved verbatim). + await db.insert('annotations', { + 'id': 'ann-1', + 'uuid': 'u-1', + 'document_id': docId, + 'page_number': 2, + 'annotation_json': '{"legacy":"blob"}', + 'created_at': '2024-01-01T00:00:00.000Z', + 'updated_at': '2024-01-01T00:00:00.000Z', + }); + + // A free-ink note with one freehand stroke (absolute px on the note page). + await db.insert('notes', { + 'id': 'note-1', + 'title': 'My Algebra Notes', + 'created_at': '2024-03-01T00:00:00.000Z', + 'updated_at': '2024-03-02T00:00:00.000Z', + 'tags': '[]', + }); + await db.insert('strokes', { + 'id': 'ns-1', + 'note_id': 'note-1', + 'tool': 'pen', + 'color': 0xFF112233, + 'stroke_width': 3.0, + 'created_at': '2024-03-01T00:00:00.000Z', + 'points': jsonEncode([ + { + 'x': 500.0, + 'y': 707.0, + 'pressure': 0.5, + 'tilt': 0.0, + 'timestamp': 0, + 'pointerDeviceKind': 'stylus', + } + ]), + }); + } finally { + await db.close(); + } + return srcPath; + } + + test('migrates a document: file copied, all tables → sidecar fields', () async { + await seedLegacy(); + + final report = await SqliteToSidecarMigrator(vault, legacyDbPath: legacyDbPath) + .run(); + + expect(report.legacyDbFound, isTrue); + expect(report.documentsMigrated, 1); + expect(report.notesMigrated, 1); + expect(report.missingSources, isEmpty); + + // The source file was copied into a notebook folder. + final docFolder = Directory(p.join(vaultDir.path, 'Lecture')); + expect(docFolder.existsSync(), isTrue); + final copiedPdf = File(p.join(docFolder.path, 'Lecture.pdf')); + expect(copiedPdf.existsSync(), isTrue); + expect(copiedPdf.readAsStringSync(), 'PDFDATA'); + + // The sidecar holds the migrated annotations. + final sidecar = await SidecarStore.read( + File(p.join(docFolder.path, 'Lecture.pdf.badnote.json')), + ); + expect(sidecar, isNotNull); + expect(sidecar!.docType, 'pdf'); + expect(sidecar.pageCount, 5); + expect(sidecar.rotation, 90); + expect(sidecar.legacyId, 'doc-1'); + + // ink → strokes[pageIndex] + expect(sidecar.strokes.keys.toSet(), {0, 3}); + expect(sidecar.strokes[0]!.single.id, 's0'); + expect(sidecar.strokes[3]!.single.id, 's3'); + + // bookmarks → bookmarks + expect(sidecar.bookmarks.single.label, 'Proof'); + expect(sidecar.bookmarks.single.pageNumber, 5); + + // scratch_links + scratchpads → scratchLinks[].link + .scratchpad + expect(sidecar.scratchLinks.single.link.id, 'anchor-1'); + expect(sidecar.scratchLinks.single.link.pageIndex, 7); + expect(sidecar.scratchLinks.single.scratchpad.strokes.single.id, + 'pad-stroke'); + + // annotations → legacyAnnotations (preserved verbatim, not dropped) + expect(sidecar.legacyAnnotations[2], '{"legacy":"blob"}'); + + // No legacy highlights → empty. + expect(sidecar.highlights, isEmpty); + }); + + test('migrates a note: standalone notebook sidecar with normalized strokes', + () async { + await seedLegacy(); + await SqliteToSidecarMigrator(vault, legacyDbPath: legacyDbPath).run(); + + final notes = await vault.scanNotes(); + expect(notes.length, 1); + expect(notes.single.title, 'My Algebra Notes'); + + final sidecar = + await SidecarStore.read(File('${notes.single.notePath}.badnote.json')); + expect(sidecar, isNotNull); + expect(sidecar!.docType, 'notebook'); + expect(sidecar.title, 'My Algebra Notes'); + expect(sidecar.legacyId, 'note-1'); + + // The legacy InkStroke (absolute px on the note logical page) was + // normalized onto page 0 exactly as the runtime note editor stores it. + final migrated = sidecar.strokes[0]!.single; + expect(migrated.id, 'ns-1'); + expect(migrated.points.first.x, closeTo(500.0 / kNoteLogicalPage.width, 1e-9)); + expect( + migrated.points.first.y, closeTo(707.0 / kNoteLogicalPage.height, 1e-9)); + }); + + test('preserves the legacy DB as *.premigration (never deletes)', () async { + await seedLegacy(); + expect(File(legacyDbPath).existsSync(), isTrue); + + final report = await SqliteToSidecarMigrator(vault, legacyDbPath: legacyDbPath) + .run(); + + // Original gone, preserved copy present. + expect(File(legacyDbPath).existsSync(), isFalse); + final preserved = File('$legacyDbPath.premigration'); + expect(preserved.existsSync(), isTrue); + expect(report.legacyDbPreservedPath, preserved.path); + // The preserved DB is non-empty (real data, not truncated). + expect(preserved.lengthSync(), greaterThan(0)); + }); + + test('is idempotent: a re-run migrates nothing new, no duplicate folders', + () async { + await seedLegacy(); + await SqliteToSidecarMigrator(vault, legacyDbPath: legacyDbPath).run(); + + // The legacy DB is now renamed; a naive second run against the ORIGINAL + // path is a no-op (file gone). But the migrator must also be idempotent if + // pointed back at the preserved DB — re-run against it and assert nothing + // duplicates. + final preservedPath = '$legacyDbPath.premigration'; + final report = await SqliteToSidecarMigrator(vault, + legacyDbPath: preservedPath) + .run(); + + expect(report.documentsMigrated, 0, reason: 'already migrated'); + expect(report.notesMigrated, 0, reason: 'already migrated'); + expect(report.documentsSkipped, 1); + expect(report.notesSkipped, 1); + + // Exactly one document folder and one note folder — no `Lecture 2` / + // `My Algebra Notes 2` duplicates. + final folders = vaultDir + .listSync() + .whereType() + .map((d) => p.basename(d.path)) + .where((n) => !n.startsWith('.')) + .toList(); + expect(folders.toSet(), {'Lecture', 'My Algebra Notes'}); + }); + + test('missing source file: annotations still migrate into a folder', () async { + await seedLegacy(createSourceFile: false); + + final report = await SqliteToSidecarMigrator(vault, legacyDbPath: legacyDbPath) + .run(); + + expect(report.documentsMigrated, 1); + expect(report.missingSources, ['Lecture.pdf']); + + // A folder exists holding ONLY the sidecar (no copied source file). + final docFolder = Directory(p.join(vaultDir.path, 'Lecture')); + expect(docFolder.existsSync(), isTrue); + expect(File(p.join(docFolder.path, 'Lecture.pdf')).existsSync(), isFalse); + + final sidecar = await SidecarStore.read( + File(p.join(docFolder.path, 'Lecture.pdf.badnote.json')), + ); + expect(sidecar, isNotNull); + expect(sidecar!.strokes.keys.toSet(), {0, 3}, reason: 'ink preserved'); + expect(sidecar.scratchLinks.single.scratchpad.strokes.single.id, + 'pad-stroke'); + }); + + test('fresh install (no legacy DB) is a safe no-op', () async { + final report = await SqliteToSidecarMigrator(vault, + legacyDbPath: p.join(tempRoot.path, 'does-not-exist.db')) + .run(); + expect(report.legacyDbFound, isFalse); + expect(report.didAnything, isFalse); + expect(report.legacyDbPreservedPath, isNull); + expect(vaultDir.listSync().whereType().length, 0); + }); + + test('throws when the vault root is invalid', () async { + await vault.setVaultRoot(p.join(tempRoot.path, 'gone$_sep')); + expect( + () => SqliteToSidecarMigrator(vault, legacyDbPath: legacyDbPath).run(), + throwsStateError, + ); + }); +}