diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 1aea4a9..fa19630 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -48,7 +48,16 @@ jobs: - name: Enable Windows desktop run: flutter config --enable-windows-desktop + # The flutter_onnxruntime plugin's CMake downloads the ONNX Runtime native + # library from github.com/microsoft/onnxruntime/releases at build time. + # That host is blocked here, but CMake's file(DOWNLOAD) honours proxy env + # vars, so we forward HTTP(S)_PROXY (set them as repo secrets, e.g. + # http://127.0.0.1:7890). Alternatively install ONNX Runtime system-wide + # and pass -DUSE_SYSTEM_ONNXRUNTIME=ON -DONNXRUNTIME_ROOT_DIR=... . - name: Build Windows release + env: + HTTP_PROXY: ${{ secrets.HTTP_PROXY }} + HTTPS_PROXY: ${{ secrets.HTTPS_PROXY }} run: flutter build windows --release - name: Package artifact diff --git a/README.md b/README.md index ebb9fc8..7b10bf3 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,9 @@ All notes, documents, search, and OCR run on your device. No server is required - Ink notes with Surface Pen (pressure, stabilizer, undo/redo) - PDF and PPT import with page-level annotation - Full-text search over note titles, typed text, and OCR results -- **Local OCR** — handwriting recognition via Windows built-in OCR (Windows desktop) +- **Local OCR** — pluggable, fully on-device. An embedded ONNX recognition + backend (cross-platform, CPU/iGPU) with a graceful fallback to the platform's + built-in OCR (Windows). See [Local OCR](#local-ocr). ## Build (Windows) @@ -60,7 +62,13 @@ lib/ │ ├── database_service.dart # SQLite + FTS5 │ ├── ocr_service.dart # Local OCR orchestration │ ├── stroke_rasterizer.dart # Ink → PNG for OCR -│ └── ocr_engine.dart # Platform OCR bridge +│ ├── ocr_engine.dart # OCR entry point (delegates to a backend) +│ └── ocr/ # Pluggable OCR backends +│ ├── ocr_backend.dart # Backend interface +│ ├── ocr_backends.dart # Backend selector (ONNX → native) +│ ├── onnx_recognition_backend.dart # Embedded ONNX (cross-platform) +│ ├── native_ocr_backend.dart # OS OCR (Windows WinRT) +│ └── ctc_decoder.dart # Pure-Dart CTC greedy decode ├── providers/ # Riverpod state └── widgets/ # Ink canvas, toolbars, thumbnails ``` @@ -69,9 +77,43 @@ OCR flow on save: 1. Extract typed text from text-tool strokes 2. Rasterize handwriting strokes to PNG -3. Run Windows OCR on the PNG +3. Recognize via the active local OCR backend (embedded ONNX if a model is + bundled, otherwise the platform's native OCR) 4. Merge recognized text into the local FTS index for search +## Local OCR + +OCR runs entirely on-device through a pluggable backend (`lib/services/ocr/`). +`OcrBackends` selects, in order: + +1. **`OnnxRecognitionBackend`** — embedded, cross-platform recognition via + `flutter_onnxruntime` (CPU/iGPU; suited to low-power APUs). Active only when + an ONNX model is bundled. +2. **`NativeOcrBackend`** — the OS built-in OCR (Windows WinRT today). + +If no backend is available, OCR is a clean no-op — the app still works. + +### Enabling the embedded ONNX model + +The model is **not committed** (it is large). Fetch it onto your dev machine +before building so it bundles as an asset: + +```bash +tool/fetch_ocr_model.sh # downloads PP-OCRv4 rec ONNX + ppocr_keys_v1.txt + # into assets/models/ocr/ (proxy hint inside) +``` + +See [assets/models/ocr/README.md](assets/models/ocr/README.md). The recognition +geometry / CTC-blank assumptions (PP-OCRv4 mobile rec, 3×48×W, blank=0) are +documented in `onnx_recognition_backend.dart` and should be verified on-device +against your exact exported model. + +> **Windows build note:** the `flutter_onnxruntime` plugin downloads the ONNX +> Runtime native library (v1.22.0) from GitHub at build time. Behind a firewall, +> set `HTTPS_PROXY` for the build (CMake honours it), or install ONNX Runtime +> system-wide and build with `-DUSE_SYSTEM_ONNXRUNTIME=ON +> -DONNXRUNTIME_ROOT_DIR=`. + ## Optional server The `server/` directory contains an experimental FastAPI backend (sync + EasyOCR). It is **not required** for the desktop app and is kept separately for future multi-device sync experiments. See [server/README.md](server/README.md). diff --git a/assets/models/ocr/.gitkeep b/assets/models/ocr/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/assets/models/ocr/README.md b/assets/models/ocr/README.md new file mode 100644 index 0000000..bbf72a1 --- /dev/null +++ b/assets/models/ocr/README.md @@ -0,0 +1,63 @@ +# Embedded OCR model (not committed) + +The handwriting/text recognition backend +(`lib/services/ocr/onnx_recognition_backend.dart`) loads an ONNX recognition +model and its character dictionary **from assets**: + +- `rec.onnx` — the PP-OCRv4 mobile text recognition model (CTC, input + `3 x 48 x W`, blank class index 0). +- `ppocr_keys_v1.txt` — the PP-OCR character dictionary, one character per line. + +Neither file is committed to the repository (the model is large and the +dictionary is distributed with PaddleOCR). The app is built to treat their +absence as a clean no-op: if the model or dictionary is missing, the ONNX +backend reports unavailable and OCR falls back to the native platform backend +(or returns nothing). Only the `.gitkeep` placeholder is committed so the +`assets/models/ocr/` asset directory is valid at build time. + +## How to obtain and place the files + +Run the helper script on your development machine (it must download from the +PaddleOCR sources and convert the Paddle inference model to ONNX): + +```bash +./tool/fetch_ocr_model.sh +``` + +This places the two files here as: + +``` +assets/models/ocr/rec.onnx +assets/models/ocr/ppocr_keys_v1.txt +``` + +### Sources + +- PP-OCRv4 mobile recognition model (PaddleOCR inference model): + https://paddleocr.bj.bcebos.com/PP-OCRv4/chinese/ch_PP-OCRv4_rec_infer.tar + (English-only variant: `en_PP-OCRv4_rec_infer.tar`) +- Character dictionary `ppocr_keys_v1.txt`: + https://raw.githubusercontent.com/PaddlePaddle/PaddleOCR/main/ppocr/utils/ppocr_keys_v1.txt + +### Conversion + +PaddleOCR ships Paddle inference models; convert to ONNX with +[paddle2onnx](https://github.com/PaddlePaddle/Paddle2ONNX): + +```bash +paddle2onnx \ + --model_dir ch_PP-OCRv4_rec_infer \ + --model_filename inference.pdmodel \ + --params_filename inference.pdiparams \ + --save_file rec.onnx \ + --opset_version 14 \ + --enable_onnx_checker True +``` + +## Verification note + +The backend assumes the PP-OCRv4 mobile rec convention (input `3 x 48 x W`, +normalization `(v/255 - 0.5)/0.5`, CTC blank at index 0, dictionary shifted by +one). If you use a different exported model, verify the input shape, +normalization, and blank/dictionary convention and adjust +`onnx_recognition_backend.dart` / `CtcDecoder` accordingly. diff --git a/lib/services/ocr/ctc_decoder.dart b/lib/services/ocr/ctc_decoder.dart new file mode 100644 index 0000000..b0237d7 --- /dev/null +++ b/lib/services/ocr/ctc_decoder.dart @@ -0,0 +1,92 @@ +/// Pure-Dart CTC (Connectionist Temporal Classification) greedy decoder. +/// +/// Decodes per-timestep class logits into a string by taking the argmax at +/// each timestep, collapsing consecutive duplicate classes, dropping the +/// blank class, and mapping the remaining class indices to characters. +/// +/// Index mapping note (PaddleOCR PP-OCR rec convention with [blankIndex] == 0): +/// the CTC blank occupies class index 0, so the character dictionary is +/// shifted by one. The character for class index `k` (k >= 1) is +/// `charset[k - 1]`. If [blankIndex] != 0, this exact shift may not apply and +/// the mapping should be reviewed for the specific exported model. +class CtcDecoder { + CtcDecoder(this.charset, {this.blankIndex = 0}); + + /// The character dictionary (without the blank entry). + final List charset; + + /// The class index reserved for the CTC blank symbol. + final int blankIndex; + + /// Decode `[T][C]` logits into a string. + /// + /// For each timestep the argmax over the `C` classes is taken; consecutive + /// duplicate indices are collapsed and the blank index is dropped. Remaining + /// indices are mapped to characters via the dictionary shift described in the + /// class docs. Out-of-range indices are skipped. + String decode(List> logits) { + final buffer = StringBuffer(); + var previousIndex = -1; + + for (final row in logits) { + if (row.isEmpty) { + previousIndex = -1; + continue; + } + + // argmax over the classes of this timestep. + var bestIndex = 0; + var bestValue = row[0]; + for (var c = 1; c < row.length; c++) { + if (row[c] > bestValue) { + bestValue = row[c]; + bestIndex = c; + } + } + + // Collapse consecutive duplicates. + if (bestIndex == previousIndex) { + continue; + } + previousIndex = bestIndex; + + // Drop the blank class. + if (bestIndex == blankIndex) { + continue; + } + + final ch = _charForIndex(bestIndex); + if (ch != null) { + buffer.write(ch); + } + } + + return buffer.toString(); + } + + /// Reshape a flat row-major `[T*C]` list into `[T][C]` and decode it. + String decodeFlat(List flat, int timeSteps, int numClasses) { + if (timeSteps <= 0 || numClasses <= 0) return ''; + final logits = >[]; + for (var t = 0; t < timeSteps; t++) { + final start = t * numClasses; + final end = start + numClasses; + if (end > flat.length) break; + logits.add(flat.sublist(start, end)); + } + return decode(logits); + } + + /// Map a class index to its character, applying the blank shift. Returns null + /// for the blank index or out-of-range indices. + String? _charForIndex(int index) { + if (index == blankIndex) return null; + + // With blankIndex == 0 the dictionary is shifted by one: class index k + // maps to charset[k - 1]. For other blank positions we fall back to a + // direct index, which may need adjustment per the exported model. + final mapped = blankIndex == 0 ? index - 1 : index; + if (mapped < 0 || mapped >= charset.length) return null; + return charset[mapped]; + } +} diff --git a/lib/services/ocr/native_ocr_backend.dart b/lib/services/ocr/native_ocr_backend.dart new file mode 100644 index 0000000..aff04c6 --- /dev/null +++ b/lib/services/ocr/native_ocr_backend.dart @@ -0,0 +1,30 @@ +import 'dart:io'; + +import 'package:flutter/services.dart'; + +import 'ocr_backend.dart'; + +/// Platform OCR backend. Uses the Windows built-in OCR engine exposed through +/// the native `badnote/ocr` MethodChannel. +class NativeOcrBackend implements OcrBackend { + static const _channel = MethodChannel('badnote/ocr'); + + @override + String get name => 'native'; + + @override + Future isAvailable() async => Platform.isWindows; + + @override + Future recognize(Uint8List pngBytes) async { + if (!Platform.isWindows) return null; + try { + final result = await _channel.invokeMethod('recognize', pngBytes); + final text = result?.trim(); + if (text == null || text.isEmpty) return null; + return text; + } catch (_) { + return null; + } + } +} diff --git a/lib/services/ocr/ocr_backend.dart b/lib/services/ocr/ocr_backend.dart new file mode 100644 index 0000000..2d10fbe --- /dev/null +++ b/lib/services/ocr/ocr_backend.dart @@ -0,0 +1,19 @@ +import 'dart:typed_data'; + +/// A pluggable local OCR backend. +/// +/// Implementations turn a PNG image into recognized text. The app selects an +/// available backend via [OcrBackends]; absence of any backend is a clean +/// no-op (recognition returns null). +abstract class OcrBackend { + /// Short identifier used for logging/selection (e.g. 'native', 'onnx'). + String get name; + + /// Whether this backend can run on the current device/build. May perform a + /// lazy initialization attempt (e.g. loading a model) the first time. + Future isAvailable(); + + /// Recognize text from a PNG image. Returns null when nothing is recognized + /// or the backend is unavailable. Implementations must never throw. + Future recognize(Uint8List pngBytes); +} diff --git a/lib/services/ocr/ocr_backends.dart b/lib/services/ocr/ocr_backends.dart new file mode 100644 index 0000000..d13184b --- /dev/null +++ b/lib/services/ocr/ocr_backends.dart @@ -0,0 +1,45 @@ +import 'dart:typed_data'; + +import 'native_ocr_backend.dart'; +import 'ocr_backend.dart'; +import 'onnx_recognition_backend.dart'; + +/// Selects and caches the active local OCR backend. +/// +/// Preference order: the embedded ONNX recognition backend if its model is +/// bundled and loads, otherwise the native platform backend, otherwise none. +class OcrBackends { + OcrBackends._(); + + static OcrBackend? _active; + static bool _resolved = false; + + /// Resolve (once) and return the preferred available backend, or null when + /// no backend is available on this device/build. + static Future active() async { + if (_resolved) return _active; + + final candidates = [ + OnnxRecognitionBackend(), + NativeOcrBackend(), + ]; + + for (final backend in candidates) { + if (await backend.isAvailable()) { + _active = backend; + break; + } + } + + _resolved = true; + return _active; + } + + /// Recognize text using the active backend. Returns null when no backend is + /// available or nothing was recognized. + static Future recognize(Uint8List png) async { + final backend = await active(); + if (backend == null) return null; + return backend.recognize(png); + } +} diff --git a/lib/services/ocr/onnx_recognition_backend.dart b/lib/services/ocr/onnx_recognition_backend.dart new file mode 100644 index 0000000..a56cb0c --- /dev/null +++ b/lib/services/ocr/onnx_recognition_backend.dart @@ -0,0 +1,204 @@ +import 'dart:typed_data'; +import 'dart:ui' as ui; + +import 'package:flutter/services.dart' show rootBundle; +import 'package:flutter_onnxruntime/flutter_onnxruntime.dart'; + +import 'ctc_decoder.dart'; +import 'ocr_backend.dart'; + +/// ONNX-based text recognition backend. +/// +/// NOTE: assumes PP-OCRv4 mobile rec (input 3x48xW, CTC blank=0). Verify +/// on-device; the dictionary/blank convention may need adjustment per the exact +/// exported model. +/// +/// The model and dictionary are bundled as assets and are optional: if either +/// is missing this backend reports unavailable and recognition is a clean +/// no-op (returns null). It never throws out of [recognize]. +class OnnxRecognitionBackend implements OcrBackend { + static const _modelAsset = 'assets/models/ocr/rec.onnx'; + static const _dictAsset = 'assets/models/ocr/ppocr_keys_v1.txt'; + + // Rec model input geometry. + static const _targetHeight = 48; + static const _minWidth = 16; + static const _maxWidth = 320; + + OrtSession? _session; + CtcDecoder? _decoder; + + bool _initAttempted = false; + bool _available = false; + + @override + String get name => 'onnx'; + + @override + Future isAvailable() async { + await _ensureInit(); + return _available; + } + + @override + Future recognize(Uint8List pngBytes) async { + await _ensureInit(); + final session = _session; + final decoder = _decoder; + if (!_available || session == null || decoder == null) return null; + + OrtValue? input; + Map? outputs; + try { + final pre = await _preprocess(pngBytes); + if (pre == null) return null; + + final inputName = session.inputNames[0]; + input = await OrtValue.fromList(pre.data, [ + 1, + 3, + _targetHeight, + pre.width, + ]); + outputs = await session.run({inputName: input}); + + final out = outputs[session.outputNames[0]]; + if (out == null) return null; + + // Expected output shape: [1, T, C]. + final shape = out.shape; + if (shape.length != 3) return null; + final timeSteps = shape[1]; + final numClasses = shape[2]; + + // asFlattenedList() returns the data flat (row-major); asList() would + // return a list nested per the output shape. + final flat = (await out.asFlattenedList()) + .map((v) => (v as num).toDouble()) + .toList(); + final text = decoder.decodeFlat(flat, timeSteps, numClasses).trim(); + if (text.isEmpty) return null; + return text; + } catch (_) { + return null; + } finally { + if (input != null) { + await input.dispose(); + } + if (outputs != null) { + for (final t in outputs.values) { + await t.dispose(); + } + } + } + } + + /// Lazily load the dictionary and create the inference session. On any + /// failure the backend is marked unavailable. + Future _ensureInit() async { + if (_initAttempted) return; + _initAttempted = true; + try { + final dictRaw = await rootBundle.loadString(_dictAsset); + final charset = dictRaw + .split('\n') + .map((line) => line.replaceAll('\r', '')) + .toList(); + // Drop a single trailing empty entry from a final newline, then append a + // space character as PP-OCR does. + if (charset.isNotEmpty && charset.last.isEmpty) { + charset.removeLast(); + } + charset.add(' '); + + final ort = OnnxRuntime(); + final session = await ort.createSessionFromAsset( + _modelAsset, + options: OrtSessionOptions( + intraOpNumThreads: 2, + providers: [OrtProvider.CPU], + ), + ); + + _session = session; + _decoder = CtcDecoder(charset, blankIndex: 0); + _available = true; + } catch (_) { + _session = null; + _decoder = null; + _available = false; + } + } + + /// Decode and preprocess the PNG into the CHW Float32 tensor the rec model + /// expects. Returns null on any decode failure. + Future<_PreprocessResult?> _preprocess(Uint8List pngBytes) async { + final codec = await ui.instantiateImageCodec(pngBytes); + final frame = await codec.getNextFrame(); + final src = frame.image; + try { + final origW = src.width; + final origH = src.height; + if (origW <= 0 || origH <= 0) return null; + + // Width that preserves aspect ratio at the target height, clamped. + final scaledW = (_targetHeight * origW / origH).round(); + final targetW = scaledW.clamp(_minWidth, _maxWidth); + + // Render the resized image onto a white canvas. If the scaled width is + // narrower than the target, the right side stays white (padding). + final recorder = ui.PictureRecorder(); + final canvas = ui.Canvas(recorder); + final paintWidth = scaledW < targetW ? scaledW : targetW; + canvas.drawRect( + ui.Rect.fromLTWH(0, 0, targetW.toDouble(), _targetHeight.toDouble()), + ui.Paint()..color = const ui.Color(0xFFFFFFFF), + ); + canvas.drawImageRect( + src, + ui.Rect.fromLTWH(0, 0, origW.toDouble(), origH.toDouble()), + ui.Rect.fromLTWH(0, 0, paintWidth.toDouble(), _targetHeight.toDouble()), + ui.Paint(), + ); + final picture = recorder.endRecording(); + final resized = await picture.toImage(targetW, _targetHeight); + picture.dispose(); + + try { + final byteData = await resized.toByteData( + format: ui.ImageByteFormat.rawRgba, + ); + if (byteData == null) return null; + final rgba = byteData.buffer.asUint8List(); + + // Layout CHW (3 x H x W), normalize (v/255 - 0.5) / 0.5, RGB only. + final hw = _targetHeight * targetW; + final data = Float32List(3 * hw); + for (var y = 0; y < _targetHeight; y++) { + for (var x = 0; x < targetW; x++) { + final pixel = (y * targetW + x) * 4; + final r = rgba[pixel] / 255.0; + final g = rgba[pixel + 1] / 255.0; + final b = rgba[pixel + 2] / 255.0; + final idx = y * targetW + x; + data[idx] = (r - 0.5) / 0.5; + data[hw + idx] = (g - 0.5) / 0.5; + data[2 * hw + idx] = (b - 0.5) / 0.5; + } + } + return _PreprocessResult(data, targetW); + } finally { + resized.dispose(); + } + } finally { + src.dispose(); + } + } +} + +class _PreprocessResult { + _PreprocessResult(this.data, this.width); + + final Float32List data; + final int width; +} diff --git a/lib/services/ocr_engine.dart b/lib/services/ocr_engine.dart index 3b19884..f9120ff 100644 --- a/lib/services/ocr_engine.dart +++ b/lib/services/ocr_engine.dart @@ -1,21 +1,14 @@ -import 'dart:io'; +import 'dart:typed_data'; -import 'package:flutter/services.dart'; +import 'ocr/ocr_backends.dart'; -/// Platform OCR backend. Uses Windows built-in OCR on desktop Windows. +/// Local OCR entry point. Delegates to a pluggable backend (embedded ONNX +/// recognition when a model is bundled, otherwise the native platform OCR). +/// +/// The static API is kept for back-compat with [OcrService]. class OcrEngine { - static const _channel = MethodChannel('badnote/ocr'); - /// Recognize text from a PNG image. Returns null when unavailable or empty. - static Future recognizeImage(Uint8List pngBytes) async { - if (!Platform.isWindows) return null; - try { - final result = await _channel.invokeMethod('recognize', pngBytes); - final text = result?.trim(); - if (text == null || text.isEmpty) return null; - return text; - } catch (_) { - return null; - } + static Future recognizeImage(Uint8List pngBytes) { + return OcrBackends.recognize(pngBytes); } } diff --git a/linux/flutter/generated_plugin_registrant.cc b/linux/flutter/generated_plugin_registrant.cc index 7299b5c..e30f888 100644 --- a/linux/flutter/generated_plugin_registrant.cc +++ b/linux/flutter/generated_plugin_registrant.cc @@ -7,12 +7,16 @@ #include "generated_plugin_registrant.h" #include +#include #include void fl_register_plugins(FlPluginRegistry* registry) { g_autoptr(FlPluginRegistrar) file_selector_linux_registrar = fl_plugin_registry_get_registrar_for_plugin(registry, "FileSelectorPlugin"); file_selector_plugin_register_with_registrar(file_selector_linux_registrar); + g_autoptr(FlPluginRegistrar) flutter_onnxruntime_registrar = + fl_plugin_registry_get_registrar_for_plugin(registry, "FlutterOnnxruntimePlugin"); + flutter_onnxruntime_plugin_register_with_registrar(flutter_onnxruntime_registrar); g_autoptr(FlPluginRegistrar) url_launcher_linux_registrar = fl_plugin_registry_get_registrar_for_plugin(registry, "UrlLauncherPlugin"); url_launcher_plugin_register_with_registrar(url_launcher_linux_registrar); diff --git a/linux/flutter/generated_plugins.cmake b/linux/flutter/generated_plugins.cmake index 886932b..6f90a96 100644 --- a/linux/flutter/generated_plugins.cmake +++ b/linux/flutter/generated_plugins.cmake @@ -4,6 +4,7 @@ list(APPEND FLUTTER_PLUGIN_LIST file_selector_linux + flutter_onnxruntime url_launcher_linux ) diff --git a/macos/Flutter/GeneratedPluginRegistrant.swift b/macos/Flutter/GeneratedPluginRegistrant.swift index 48b3875..bf51a80 100644 --- a/macos/Flutter/GeneratedPluginRegistrant.swift +++ b/macos/Flutter/GeneratedPluginRegistrant.swift @@ -8,6 +8,7 @@ import Foundation import device_info_plus import file_picker import file_selector_macos +import flutter_onnxruntime import shared_preferences_foundation import sqflite_darwin import syncfusion_pdfviewer_macos @@ -17,6 +18,7 @@ func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) { DeviceInfoPlusMacosPlugin.register(with: registry.registrar(forPlugin: "DeviceInfoPlusMacosPlugin")) FilePickerPlugin.register(with: registry.registrar(forPlugin: "FilePickerPlugin")) FileSelectorPlugin.register(with: registry.registrar(forPlugin: "FileSelectorPlugin")) + FlutterOnnxruntimePlugin.register(with: registry.registrar(forPlugin: "FlutterOnnxruntimePlugin")) SharedPreferencesPlugin.register(with: registry.registrar(forPlugin: "SharedPreferencesPlugin")) SqflitePlugin.register(with: registry.registrar(forPlugin: "SqflitePlugin")) SyncfusionFlutterPdfViewerPlugin.register(with: registry.registrar(forPlugin: "SyncfusionFlutterPdfViewerPlugin")) diff --git a/pubspec.lock b/pubspec.lock index 6e7bcb4..c54befd 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -326,6 +326,14 @@ packages: url: "https://pub.dev" source: hosted version: "6.0.0" + flutter_onnxruntime: + dependency: "direct main" + description: + name: flutter_onnxruntime + sha256: "616f0e296840edb63278c647baf4475232d21cba45478ff26348e474f3f6a913" + url: "https://pub.dev" + source: hosted + version: "1.8.0" flutter_plugin_android_lifecycle: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 0bbde4e..a5912fa 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -48,6 +48,9 @@ dependencies: # Camera / image picker image_picker: ^1.1.2 + # Embedded ONNX runtime (local OCR recognition backend) + flutter_onnxruntime: ^1.8.0 + dev_dependencies: flutter_test: sdk: flutter @@ -62,6 +65,9 @@ dev_dependencies: flutter: uses-material-design: true + assets: + - assets/models/ocr/ + # Use vendored, hash-verified sqlite3 native binaries (committed under # vendor/sqlite3/) instead of downloading them from GitHub releases at build # time. This keeps builds fully local/offline — important behind the GFW where diff --git a/test/ctc_decoder_test.dart b/test/ctc_decoder_test.dart new file mode 100644 index 0000000..36e8c4c --- /dev/null +++ b/test/ctc_decoder_test.dart @@ -0,0 +1,75 @@ +import 'package:badnote/services/ocr/ctc_decoder.dart'; +import 'package:flutter_test/flutter_test.dart'; + +/// Build a one-hot-ish logits row of [numClasses] with the max at [maxIndex]. +List _row(int numClasses, int maxIndex) { + return List.generate(numClasses, (i) => i == maxIndex ? 1.0 : 0.0); +} + +void main() { + group('CtcDecoder.decode', () { + test('collapses consecutive repeats and drops blanks (+1 shift)', () { + // charset indices: 1->'a', 2->'b', 3->'c' (blank at 0, shifted by one). + final decoder = CtcDecoder(['a', 'b', 'c'], blankIndex: 0); + const numClasses = 4; // blank + 3 chars + final logits = >[ + _row(numClasses, 1), // a + _row(numClasses, 1), // a (collapsed) + _row(numClasses, 0), // blank + _row(numClasses, 2), // b + _row(numClasses, 2), // b (collapsed) + _row(numClasses, 3), // c + ]; + expect(decoder.decode(logits), 'abc'); + }); + + test('empty input yields empty string', () { + final decoder = CtcDecoder(['a', 'b', 'c'], blankIndex: 0); + expect(decoder.decode(>[]), ''); + }); + + test('all-blank input yields empty string', () { + final decoder = CtcDecoder(['a', 'b', 'c'], blankIndex: 0); + const numClasses = 4; + final logits = >[ + _row(numClasses, 0), + _row(numClasses, 0), + _row(numClasses, 0), + ]; + expect(decoder.decode(logits), ''); + }); + + test('out-of-range indices are skipped', () { + // charset has 2 entries -> valid class indices are 1 and 2. Class index 3 + // maps to charset[2] which is out of range and must be skipped. + final decoder = CtcDecoder(['a', 'b'], blankIndex: 0); + const numClasses = 4; + final logits = >[ + _row(numClasses, 1), // a + _row(numClasses, 3), // out of range -> skipped + _row(numClasses, 2), // b + ]; + expect(decoder.decode(logits), 'ab'); + }); + }); + + group('CtcDecoder.decodeFlat', () { + test('reshapes a flat row-major list and decodes it', () { + final decoder = CtcDecoder(['a', 'b', 'c'], blankIndex: 0); + const numClasses = 4; + const timeSteps = 3; + final flat = [ + ..._row(numClasses, 1), // a + ..._row(numClasses, 0), // blank + ..._row(numClasses, 2), // b + ]; + expect(decoder.decodeFlat(flat, timeSteps, numClasses), 'ab'); + }); + + test('returns empty for non-positive dimensions', () { + final decoder = CtcDecoder(['a'], blankIndex: 0); + expect(decoder.decodeFlat([1, 0], 0, 2), ''); + expect(decoder.decodeFlat([1, 0], 2, 0), ''); + }); + }); +} diff --git a/tool/fetch_ocr_model.sh b/tool/fetch_ocr_model.sh new file mode 100755 index 0000000..8b17a2e --- /dev/null +++ b/tool/fetch_ocr_model.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# fetch_ocr_model.sh — download and prepare the embedded OCR recognition model. +# +# RUN THIS ON YOUR DEV MACHINE. It downloads the PaddleOCR PP-OCRv4 mobile +# recognition inference model + the character dictionary, converts the Paddle +# inference model to ONNX, and places the results as: +# +# assets/models/ocr/rec.onnx +# assets/models/ocr/ppocr_keys_v1.txt +# +# These files are intentionally NOT committed; the app treats their absence as +# a clean no-op (OCR falls back to the native backend or returns nothing). +# +# Requirements: bash, curl, tar, and paddle2onnx (pip install paddle2onnx). +# +# proxy: export HTTPS_PROXY=http://127.0.0.1:7890 (and HTTP_PROXY) if you are +# behind a firewall/GFW that blocks the download hosts. + +set -euo pipefail + +# Resolve repo root relative to this script so it works from any cwd. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +OUT_DIR="${REPO_ROOT}/assets/models/ocr" +WORK_DIR="$(mktemp -d)" + +# Canonical PaddleOCR sources. Swap to the en_ variant for English-only. +REC_INFER_URL="https://paddleocr.bj.bcebos.com/PP-OCRv4/chinese/ch_PP-OCRv4_rec_infer.tar" +# REC_INFER_URL="https://paddleocr.bj.bcebos.com/PP-OCRv4/english/en_PP-OCRv4_rec_infer.tar" +KEYS_URL="https://raw.githubusercontent.com/PaddlePaddle/PaddleOCR/main/ppocr/utils/ppocr_keys_v1.txt" + +cleanup() { rm -rf "${WORK_DIR}"; } +trap cleanup EXIT + +mkdir -p "${OUT_DIR}" + +echo "==> Downloading recognition inference model" +curl -fL "${REC_INFER_URL}" -o "${WORK_DIR}/rec_infer.tar" + +echo "==> Extracting" +tar -xf "${WORK_DIR}/rec_infer.tar" -C "${WORK_DIR}" +# The tarball extracts into a single directory; find it. +MODEL_DIR="$(find "${WORK_DIR}" -maxdepth 1 -type d -name '*_rec_infer' | head -n1)" +if [[ -z "${MODEL_DIR}" ]]; then + echo "ERROR: could not locate the extracted *_rec_infer directory" >&2 + exit 1 +fi + +echo "==> Downloading character dictionary" +curl -fL "${KEYS_URL}" -o "${OUT_DIR}/ppocr_keys_v1.txt" + +echo "==> Converting Paddle inference model to ONNX (requires paddle2onnx)" +if ! command -v paddle2onnx >/dev/null 2>&1; then + echo "ERROR: paddle2onnx not found. Install with: pip install paddle2onnx" >&2 + exit 1 +fi + +paddle2onnx \ + --model_dir "${MODEL_DIR}" \ + --model_filename inference.pdmodel \ + --params_filename inference.pdiparams \ + --save_file "${OUT_DIR}/rec.onnx" \ + --opset_version 14 \ + --enable_onnx_checker True + +echo "==> Done:" +echo " ${OUT_DIR}/rec.onnx" +echo " ${OUT_DIR}/ppocr_keys_v1.txt" diff --git a/windows/flutter/generated_plugin_registrant.cc b/windows/flutter/generated_plugin_registrant.cc index 779f0ee..f75d9c3 100644 --- a/windows/flutter/generated_plugin_registrant.cc +++ b/windows/flutter/generated_plugin_registrant.cc @@ -7,12 +7,15 @@ #include "generated_plugin_registrant.h" #include +#include #include #include void RegisterPlugins(flutter::PluginRegistry* registry) { FileSelectorWindowsRegisterWithRegistrar( registry->GetRegistrarForPlugin("FileSelectorWindows")); + FlutterOnnxruntimePluginRegisterWithRegistrar( + registry->GetRegistrarForPlugin("FlutterOnnxruntimePlugin")); SyncfusionPdfviewerWindowsPluginRegisterWithRegistrar( registry->GetRegistrarForPlugin("SyncfusionPdfviewerWindowsPlugin")); UrlLauncherWindowsRegisterWithRegistrar( diff --git a/windows/flutter/generated_plugins.cmake b/windows/flutter/generated_plugins.cmake index 2e6e95f..cf85c27 100644 --- a/windows/flutter/generated_plugins.cmake +++ b/windows/flutter/generated_plugins.cmake @@ -4,6 +4,7 @@ list(APPEND FLUTTER_PLUGIN_LIST file_selector_windows + flutter_onnxruntime syncfusion_pdfviewer_windows url_launcher_windows )