From 0cf58fb67a629c15a4b41eabcb22b401d307be6e Mon Sep 17 00:00:00 2001
From: Akiba So <given@2890.ltd>
Date: Tue, 23 Jun 2026 03:30:42 +0800
Subject: [PATCH] feat(f8): CJK-safe search text normalization + matching
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

normalizeForIndex (lowercase + collapse whitespace runs incl. hard PDF/OCR
newlines + trim) and matchesNormalized so a query matches across the line breaks
in raw extracted text. Deliberately NO word-tokenization: Chinese has no
inter-word spaces, so substring match over normalized text is correct for both
Latin and CJK (段/word segmentation belongs in the DB FTS tokenizer). Verified
on CJK inputs (你好/笔记应用).

flutter analyze lib/editor clean; 181/181 tests (+9).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 lib/editor/search/search_text.dart | 30 ++++++++++++++++++
 test/search_text_test.dart         | 51 ++++++++++++++++++++++++++++++
 2 files changed, 81 insertions(+)
 create mode 100644 lib/editor/search/search_text.dart
 create mode 100644 test/search_text_test.dart

diff --git a/lib/editor/search/search_text.dart b/lib/editor/search/search_text.dart
new file mode 100644
index 0000000..9a980e4
--- /dev/null
+++ b/lib/editor/search/search_text.dart
@@ -0,0 +1,30 @@
+// lib/editor/search/search_text.dart
+//
+// Pure text normalization + matching for full-text search (F8). PDF text layers
+// and OCR output are full of hard line breaks and irregular whitespace, so a
+// query like "hello world" won't substring-match raw extracted text that reads
+// "hello\nworld". Normalizing both sides (lowercase + collapse every whitespace
+// run to a single space + trim) fixes that.
+//
+// CJK NOTE: this user writes Chinese. We deliberately do NOT word-tokenize —
+// Chinese has no inter-word spaces, so a whitespace/punctuation tokenizer would
+// mangle it. Substring matching over normalized text is correct for both Latin
+// and CJK; word/段 segmentation belongs in the DB FTS tokenizer (trigram /
+// unicode61), not here.
+
+/// Matches any run of Unicode whitespace (spaces, tabs, newlines, NBSP, …).
+final RegExp _whitespaceRun = RegExp(r'\s+');
+
+/// Normalize [text] for indexing/matching: lowercase, collapse whitespace runs
+/// (incl. the hard newlines PDF/OCR insert mid-sentence) to single spaces, trim.
+String normalizeForIndex(String text) {
+  return text.toLowerCase().replaceAll(_whitespaceRun, ' ').trim();
+}
+
+/// Whether [source] contains [query] after both are normalized — so a match can
+/// span the line breaks present in the raw text. Empty query never matches.
+bool matchesNormalized(String source, String query) {
+  final q = normalizeForIndex(query);
+  if (q.isEmpty) return false;
+  return normalizeForIndex(source).contains(q);
+}
diff --git a/test/search_text_test.dart b/test/search_text_test.dart
new file mode 100644
index 0000000..403dbf0
--- /dev/null
+++ b/test/search_text_test.dart
@@ -0,0 +1,51 @@
+// Tests for CJK-safe search text normalization + matching (F8).
+
+import 'package:flutter_test/flutter_test.dart';
+
+import 'package:badnote/editor/search/search_text.dart';
+
+void main() {
+  group('normalizeForIndex', () {
+    test('lowercases and collapses whitespace runs to single spaces', () {
+      expect(normalizeForIndex('Hello   World'), 'hello world');
+      expect(normalizeForIndex('a\t b\n\nc'), 'a b c');
+    });
+
+    test('trims leading/trailing whitespace', () {
+      expect(normalizeForIndex('  padded  '), 'padded');
+    });
+
+    test('collapses hard newlines from PDF/OCR mid-sentence', () {
+      expect(normalizeForIndex('hello\nworld'), 'hello world');
+    });
+
+    test('leaves CJK intact (no tokenization/mangling)', () {
+      expect(normalizeForIndex('你好  世界'), '你好 世界');
+      expect(normalizeForIndex('笔记\n应用'), '笔记 应用');
+    });
+  });
+
+  group('matchesNormalized', () {
+    test('matches across a line break in the source', () {
+      expect(matchesNormalized('hello\nworld', 'hello world'), isTrue);
+    });
+
+    test('is case-insensitive', () {
+      expect(matchesNormalized('The Quick Fox', 'quick'), isTrue);
+    });
+
+    test('CJK substring match works', () {
+      expect(matchesNormalized('这是一个笔记应用', '笔记'), isTrue);
+      expect(matchesNormalized('这是一个笔记\n应用', '笔记 应用'), isTrue);
+    });
+
+    test('empty query never matches', () {
+      expect(matchesNormalized('anything', ''), isFalse);
+      expect(matchesNormalized('anything', '   '), isFalse);
+    });
+
+    test('non-match returns false', () {
+      expect(matchesNormalized('hello world', 'zzz'), isFalse);
+    });
+  });
+}