feat(storage): app-pause flush + vault search index
Some checks failed
CI / Windows build (push) Has been cancelled
Some checks failed
CI / Windows build (push) Has been cancelled
Phase 6 (final storage phase). - SidecarRepositoryRegistry tracks every open repo; SidecarFlushObserver (a WidgetsBindingObserver in main) flushes them all on inactive/hidden/paused/detached, awaiting each flush — the last strokes can't be lost on app close, not just on the 800ms timer. - VaultSearchIndex rebuilds by scanning vault sidecars (the source of truth) — note titles, OCR text and document names — and search_provider queries it, so search spans notes + PDFs. Rebuilt on launch / after import. The vault file-based storage migration (Phases 0-6) is complete: annotations travel with the file, picked vault folder, atomic autosave, one Import-file entry, SQLite migrated to sidecars. analyze clean, tests green.
This commit is contained in:
@@ -1,3 +1,6 @@
|
||||
import 'dart:io';
|
||||
|
||||
import '../editor/persistence/sidecar_repository.dart';
|
||||
import '../models/note.dart';
|
||||
import '../models/pen_tool.dart';
|
||||
import 'database_service.dart';
|
||||
@@ -6,7 +9,12 @@ import 'stroke_rasterizer.dart';
|
||||
|
||||
/// Runs OCR locally: typed text from strokes + handwriting via platform OCR.
|
||||
class OcrService {
|
||||
/// Extract searchable text from [note] and merge into the local FTS index.
|
||||
/// Extract searchable text from [note] and persist it for search. The text is
|
||||
/// written to the note's `*.badnote.json` sidecar `ocrText` field — the
|
||||
/// vault-scan source of truth the Phase 6 search index reads — and also
|
||||
/// appended to the (rebuildable) SQLite FTS cache so legacy callers keep
|
||||
/// working. [note.id] is the synthetic note path, which is exactly the
|
||||
/// `sourceFilePath` the editor opened its [SidecarRepository] with.
|
||||
Future<void> processNote(Note note) async {
|
||||
final parts = <String>[];
|
||||
|
||||
@@ -40,6 +48,22 @@ class OcrService {
|
||||
final combined = parts.join(' ').trim();
|
||||
if (combined.isEmpty) return;
|
||||
|
||||
// Persist into the note's sidecar so the vault-scan search index finds it.
|
||||
// Prefer the editor's already-open repo (same in-memory sidecar — no race);
|
||||
// if the note is closed, open/flush/dispose a transient handle.
|
||||
final open = SidecarRepositoryRegistry.forPath(note.id);
|
||||
if (open != null) {
|
||||
open.scheduleOcrTextSave(combined);
|
||||
await open.flush();
|
||||
} else if (await File('${note.id}$kSidecarSuffix').exists()) {
|
||||
final repo = await SidecarRepository.open(note.id, docType: 'notebook');
|
||||
repo.scheduleOcrTextSave(combined);
|
||||
await repo.flush();
|
||||
repo.dispose();
|
||||
}
|
||||
|
||||
// Also keep the legacy SQLite FTS cache warm (rebuildable; not the source
|
||||
// of truth). Harmless if the row is never read.
|
||||
final db = await DatabaseService.getInstance();
|
||||
await db.appendOcrToFts(note.id, combined);
|
||||
}
|
||||
|
||||
180
lib/services/vault_search_index.dart
Normal file
180
lib/services/vault_search_index.dart
Normal file
@@ -0,0 +1,180 @@
|
||||
// lib/services/vault_search_index.dart
|
||||
//
|
||||
// Phase 6 of the file-based storage plan (docs/plans/2026-06-24-file-based-
|
||||
// storage.md §B/§F): the search index, rebuilt by SCANNING the vault sidecars
|
||||
// (the source of truth) rather than the demoted SQLite cache.
|
||||
//
|
||||
// What it indexes, per notebook/note folder, from its `*.badnote.json` sidecar:
|
||||
// * the title (standalone notebooks) / source filename (file-backed docs),
|
||||
// * every typed text box — EditorStroke(tool: text).textContent across all
|
||||
// pages,
|
||||
// * the handwriting OCR text persisted in the sidecar's `ocrText` field.
|
||||
//
|
||||
// It does NOT (yet) index a PDF's embedded text layer — that is a known gap (see
|
||||
// the REPORT in the task / the class doc below). Matching uses the existing pure
|
||||
// search primitives (normalize / rank / snippet), so CJK substring search works.
|
||||
|
||||
import 'dart:io';
|
||||
|
||||
import '../editor/search/search_ranking.dart';
|
||||
import '../editor/search/search_snippet.dart';
|
||||
import '../storage/badnote_sidecar.dart';
|
||||
import '../storage/sidecar_store.dart';
|
||||
import 'vault_service.dart';
|
||||
|
||||
/// One indexed notebook: where it lives and the text harvested from its sidecar.
|
||||
class VaultSearchEntry {
|
||||
const VaultSearchEntry({
|
||||
required this.id,
|
||||
required this.title,
|
||||
required this.openPath,
|
||||
required this.docType,
|
||||
required this.text,
|
||||
required this.isNote,
|
||||
});
|
||||
|
||||
/// Stable id (the notebook folder path).
|
||||
final String id;
|
||||
|
||||
/// Display title (note title / source filename).
|
||||
final String title;
|
||||
|
||||
/// Path to pass to the editor: the in-vault source file for docs, or the
|
||||
/// synthetic `<folder>/notebook` note path for standalone notebooks.
|
||||
final String openPath;
|
||||
|
||||
/// `pdf` / `pptx` / `ppt` / `docx` / `notebook`.
|
||||
final String docType;
|
||||
|
||||
/// All searchable text harvested from the sidecar (title + typed text + OCR),
|
||||
/// joined for substring matching.
|
||||
final String text;
|
||||
|
||||
/// True for standalone (free-ink) notebooks, false for file-backed documents.
|
||||
final bool isNote;
|
||||
}
|
||||
|
||||
/// A search hit over the vault: the entry plus a display snippet of the match.
|
||||
class VaultSearchHit {
|
||||
const VaultSearchHit({required this.entry, required this.snippet});
|
||||
|
||||
final VaultSearchEntry entry;
|
||||
final Snippet snippet;
|
||||
}
|
||||
|
||||
/// Builds and queries a scan-based full-text index over the vault sidecars.
|
||||
///
|
||||
/// The index is the list of [VaultSearchEntry]s built by [rebuild]; query is a
|
||||
/// pure substring/rank over their harvested text (CJK-safe). Cheap enough to
|
||||
/// rebuild lazily on demand for a single-user vault; there is no background
|
||||
/// thread and no persisted index file (the SQLite cache is no longer the search
|
||||
/// source of truth).
|
||||
///
|
||||
/// HONEST SCOPE (what search covers / does NOT):
|
||||
/// * COVERS: note/doc titles, typed text boxes, and handwriting OCR text that
|
||||
/// has been persisted into a sidecar's `ocrText` field.
|
||||
/// * DOES NOT cover: a PDF's embedded (printed) text layer — only the user's
|
||||
/// annotations are indexed, not the underlying document body. Indexing the
|
||||
/// PDF text layer would require rendering each page through pdfrx at scan
|
||||
/// time; deferred. OCR is only present where it has already been run and
|
||||
/// written back to the sidecar.
|
||||
class VaultSearchIndex {
|
||||
VaultSearchIndex(this._vault);
|
||||
|
||||
final VaultService _vault;
|
||||
|
||||
List<VaultSearchEntry> _entries = const [];
|
||||
bool _built = false;
|
||||
|
||||
/// The entries from the most recent [rebuild] (for tests / inspection).
|
||||
List<VaultSearchEntry> get entries => List.unmodifiable(_entries);
|
||||
|
||||
/// Scan the vault and (re)build the in-memory index. Safe on an empty/missing
|
||||
/// vault (yields an empty index). Never throws on a single unreadable sidecar.
|
||||
Future<void> rebuild() async {
|
||||
final entries = <VaultSearchEntry>[];
|
||||
|
||||
final notebooks = await _vault.scanNotebooks();
|
||||
for (final nb in notebooks) {
|
||||
final sidecar = await SidecarStore.read(
|
||||
File('${nb.sourceFilePath}$kVaultSidecarSuffix'),
|
||||
);
|
||||
entries.add(
|
||||
VaultSearchEntry(
|
||||
id: nb.folderPath,
|
||||
title: nb.filename,
|
||||
openPath: nb.sourceFilePath,
|
||||
docType: nb.docType,
|
||||
text: _harvest(title: nb.filename, sidecar: sidecar),
|
||||
isNote: false,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
final notes = await _vault.scanNotes();
|
||||
for (final note in notes) {
|
||||
final sidecar = await SidecarStore.read(
|
||||
File('${note.notePath}$kVaultSidecarSuffix'),
|
||||
);
|
||||
entries.add(
|
||||
VaultSearchEntry(
|
||||
id: note.folderPath,
|
||||
title: note.title,
|
||||
openPath: note.notePath,
|
||||
docType: 'notebook',
|
||||
text: _harvest(title: note.title, sidecar: sidecar),
|
||||
isNote: true,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
_entries = entries;
|
||||
_built = true;
|
||||
}
|
||||
|
||||
/// Search the index for [query], rebuilding it first if it has never been
|
||||
/// built. Returns the best-matching notebooks/notes, most-relevant first. An
|
||||
/// empty/whitespace query yields no hits.
|
||||
Future<List<VaultSearchHit>> search(String query) async {
|
||||
if (query.trim().isEmpty) return const [];
|
||||
if (!_built) await rebuild();
|
||||
|
||||
final sources = <String, String>{
|
||||
for (final e in _entries) e.id: e.text,
|
||||
};
|
||||
final ranked = rankHits(sources, query);
|
||||
final byId = {for (final e in _entries) e.id: e};
|
||||
|
||||
final hits = <VaultSearchHit>[];
|
||||
for (final hit in ranked) {
|
||||
final entry = byId[hit.ref];
|
||||
if (entry == null) continue;
|
||||
hits.add(VaultSearchHit(entry: entry, snippet: hit.snippet));
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
|
||||
/// Concatenate every searchable string from one sidecar: the [title], every
|
||||
/// typed text box across all pages, and the persisted handwriting OCR text.
|
||||
static String _harvest({
|
||||
required String title,
|
||||
required BadnoteSidecar? sidecar,
|
||||
}) {
|
||||
final parts = <String>[title];
|
||||
if (sidecar != null) {
|
||||
// Typed text boxes: a stroke carries `textContent` regardless of tool
|
||||
// (EditorTool has only pen/highlighter/eraser; text is a content flag).
|
||||
for (final pageStrokes in sidecar.strokes.values) {
|
||||
for (final stroke in pageStrokes) {
|
||||
final text = stroke.textContent;
|
||||
if (text != null && text.trim().isNotEmpty) {
|
||||
parts.add(text.trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
final ocr = sidecar.ocrText;
|
||||
if (ocr != null && ocr.trim().isNotEmpty) parts.add(ocr.trim());
|
||||
}
|
||||
return parts.join('\n');
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user