From 219f2b5eb2719b213635f400cbaba9ba3e53e76e Mon Sep 17 00:00:00 2001 From: viwe-monai Date: Wed, 30 Sep 2026 22:45:36 +0800 Subject: [PATCH 1/3] fix: expand typographic ligatures instead of dropping them (#172) Unicode ligatures (U+FB00-FB06: ff, fi, fl, ffi, ffl, st) were dropped entirely during text extraction, silently corrupting words (classifies -> 'classi es', affected -> 'a ected'). Expand them to their constituent ASCII letters in clean_text() and add a regression test covering all six code points. Fixes #172 --- src/shared/text.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/shared/text.rs b/src/shared/text.rs index aa151948..e61afb9c 100644 --- a/src/shared/text.rs +++ b/src/shared/text.rs @@ -4,6 +4,8 @@ /// regular space, strip soft hyphens. Line breaks become single spaces; a /// CRLF pair is one break. U+200C (ZWNJ) and U+200D (ZWJ) are **preserved**: /// they carry meaning in Arabic/Indic shaping and emoji sequences. +/// Typographic ligatures (ff, fi, fl, ffi, ffl, st) are expanded to their +/// constituent letters so words are not silently corrupted. pub fn clean_text(text: &str) -> String { let mut out = String::with_capacity(text.len()); let mut chars = text.chars().peekable(); @@ -19,6 +21,13 @@ pub fn clean_text(text: &str) -> String { out.push(' '); } '\n' => out.push(' '), + // Expand typographic ligatures to ASCII equivalents. + '\u{fb00}' => out.push_str("ff"), + '\u{fb01}' => out.push_str("fi"), + '\u{fb02}' => out.push_str("fl"), + '\u{fb03}' => out.push_str("ffi"), + '\u{fb04}' => out.push_str("ffl"), + '\u{fb05}' | '\u{fb06}' => out.push_str("st"), c if c.is_control() => {} c => out.push(c), } @@ -58,4 +67,13 @@ mod tests { fn layout_invisibles_stripped() { assert_eq!(clean_text("a\u{ad}b\u{200b}c\u{feff}d\u{a0}e"), "abcd e"); } + + #[test] + fn ligatures_expanded() { + assert_eq!(clean_text("classi\u{fb01}es"), "classifies"); + assert_eq!(clean_text("a\u{fb00}ected"), "affected"); + assert_eq!(clean_text("de\u{fb02}ect"), "deflect"); + assert_eq!(clean_text("di\u{fb03}cult"), "difficult"); + assert_eq!(clean_text("waf\u{fb04}e"), "waffle"); + } } From 312b6c65e48a2f9014deab9e0bdadf2303b77e60 Mon Sep 17 00:00:00 2001 From: viwe-monai Date: Wed, 30 Sep 2026 23:07:25 +0800 Subject: [PATCH 2/3] fix: apply ligature expansion on the PDF extraction path (#185 review) Address review feedback: - P2: The PDF path (formats::pdf::to_markdown) bypasses clean_text(). Add expand_ligatures() in the PDF module and apply it to the extracted markdown before returning. - P3: Add U+FB05 and U+FB06 test coverage to ligatures_expanded. --- src/formats/pdf.rs | 25 ++++++++++++++++++++++++- src/shared/text.rs | 2 ++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/formats/pdf.rs b/src/formats/pdf.rs index 7bc7de23..3699676c 100644 --- a/src/formats/pdf.rs +++ b/src/formats/pdf.rs @@ -11,6 +11,28 @@ use crate::error::ConvertError; use pdf_inspector::PdfError; +/// Expand typographic ligatures to their constituent ASCII letters. +/// Applied to PDF output because the PDF extraction path bypasses the +/// shared `clean_text` normalizer. +fn expand_ligatures(text: &str) -> String { + if !text.contains('\u{fb00}'..='\u{fb06}') { + return text.to_string(); + } + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + match c { + '\u{fb00}' => out.push_str("ff"), + '\u{fb01}' => out.push_str("fi"), + '\u{fb02}' => out.push_str("fl"), + '\u{fb03}' => out.push_str("ffi"), + '\u{fb04}' => out.push_str("ffl"), + '\u{fb05}' | '\u{fb06}' => out.push_str("st"), + c => out.push(c), + } + } + out +} + pub fn to_markdown(bytes: &[u8]) -> Result { let result = pdf_inspector::process_pdf_mem(bytes).map_err(map_error)?; if !result.pages_needing_ocr.is_empty() { @@ -28,7 +50,8 @@ pub fn to_markdown(bytes: &[u8]) -> Result { log::warn!("broken font encodings detected; extracted text may be garbled"); } match result.markdown { - Some(mut markdown) if !markdown.trim().is_empty() => { + Some(markdown) if !markdown.trim().is_empty() => { + let mut markdown = expand_ligatures(&markdown); if !markdown.ends_with('\n') { markdown.push('\n'); } diff --git a/src/shared/text.rs b/src/shared/text.rs index e61afb9c..c4cf5ac3 100644 --- a/src/shared/text.rs +++ b/src/shared/text.rs @@ -75,5 +75,7 @@ mod tests { assert_eq!(clean_text("de\u{fb02}ect"), "deflect"); assert_eq!(clean_text("di\u{fb03}cult"), "difficult"); assert_eq!(clean_text("waf\u{fb04}e"), "waffle"); + assert_eq!(clean_text("fa\u{fb05}"), "fast"); + assert_eq!(clean_text("ju\u{fb06}"), "just"); } } From 0686d87c2327423b0c9de17fa2d16ef5d0903794 Mon Sep 17 00:00:00 2001 From: viwe-monai Date: Sat, 24 Oct 2026 13:12:23 +0800 Subject: [PATCH 3/3] fix: use a stable Pattern for the ligature fast path (#185 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review feedback from @rahul05ranjan: - The fast path `text.contains('\u{fb00}'..='\u{fb06}')` does not compile on stable Rust: `str::contains` takes a `Pattern` and `RangeInclusive` only implements `Pattern` on nightly (E0277, reproduced with rustc 1.99.1). Use `chars().any(...)` instead. The range endpoints U+FB00–U+FB06 match the expansion table exactly (ff fi fl ffi ffl st st), so the set of expanded characters is unchanged. - Deduplicate the ligature table: `expand_ligatures` is now `pub(crate)` in `shared/text.rs`, backed by a per-char `ligature_expansion` helper that `clean_text` also uses. pdf.rs keeps its own call site because `clean_text` collapses line breaks, which PDF Markdown must keep. - Fix a latent test typo present since 219f2b5: `clean_text("waf\u{fb04}e")` cannot yield "waffle" because U+FB04 expands to "ffl"; the ligature form of "waffle" is `"wa\u{fb04}e"`. - Add `expand_ligatures_pdf_path` covering all seven ligature code points, line-break preservation, and the no-ligature fast path. Verified locally on stable (rustc 1.99.1), mirroring CI: cargo fmt --all --check, cargo clippy --workspace --all-targets --all-features -- -D warnings, cargo clippy -p anydoc-wasm --target wasm32-unknown-unknown, and cargo test --locked (all green). --- src/formats/pdf.rs | 23 +---------------- src/shared/text.rs | 64 +++++++++++++++++++++++++++++++++++++++------- 2 files changed, 56 insertions(+), 31 deletions(-) diff --git a/src/formats/pdf.rs b/src/formats/pdf.rs index 3699676c..c521ccdc 100644 --- a/src/formats/pdf.rs +++ b/src/formats/pdf.rs @@ -9,30 +9,9 @@ //! [pdf-inspector]: https://github.com/firecrawl/pdf-inspector use crate::error::ConvertError; +use crate::shared::text::expand_ligatures; use pdf_inspector::PdfError; -/// Expand typographic ligatures to their constituent ASCII letters. -/// Applied to PDF output because the PDF extraction path bypasses the -/// shared `clean_text` normalizer. -fn expand_ligatures(text: &str) -> String { - if !text.contains('\u{fb00}'..='\u{fb06}') { - return text.to_string(); - } - let mut out = String::with_capacity(text.len()); - for c in text.chars() { - match c { - '\u{fb00}' => out.push_str("ff"), - '\u{fb01}' => out.push_str("fi"), - '\u{fb02}' => out.push_str("fl"), - '\u{fb03}' => out.push_str("ffi"), - '\u{fb04}' => out.push_str("ffl"), - '\u{fb05}' | '\u{fb06}' => out.push_str("st"), - c => out.push(c), - } - } - out -} - pub fn to_markdown(bytes: &[u8]) -> Result { let result = pdf_inspector::process_pdf_mem(bytes).map_err(map_error)?; if !result.pages_needing_ocr.is_empty() { diff --git a/src/shared/text.rs b/src/shared/text.rs index c4cf5ac3..46e94549 100644 --- a/src/shared/text.rs +++ b/src/shared/text.rs @@ -10,6 +10,11 @@ pub fn clean_text(text: &str) -> String { let mut out = String::with_capacity(text.len()); let mut chars = text.chars().peekable(); while let Some(c) = chars.next() { + // Expand typographic ligatures to ASCII equivalents. + if let Some(expansion) = ligature_expansion(c) { + out.push_str(expansion); + continue; + } match c { '\u{a0}' => out.push(' '), '\u{ad}' | '\u{200b}' | '\u{feff}' => {} @@ -21,13 +26,6 @@ pub fn clean_text(text: &str) -> String { out.push(' '); } '\n' => out.push(' '), - // Expand typographic ligatures to ASCII equivalents. - '\u{fb00}' => out.push_str("ff"), - '\u{fb01}' => out.push_str("fi"), - '\u{fb02}' => out.push_str("fl"), - '\u{fb03}' => out.push_str("ffi"), - '\u{fb04}' => out.push_str("ffl"), - '\u{fb05}' | '\u{fb06}' => out.push_str("st"), c if c.is_control() => {} c => out.push(c), } @@ -35,6 +33,42 @@ pub fn clean_text(text: &str) -> String { out } +/// The ASCII expansion for a typographic ligature (U+FB00–U+FB06: +/// ff fi fl ffi ffl ſt st), or `None` for any other character. +fn ligature_expansion(c: char) -> Option<&'static str> { + match c { + '\u{fb00}' => Some("ff"), + '\u{fb01}' => Some("fi"), + '\u{fb02}' => Some("fl"), + '\u{fb03}' => Some("ffi"), + '\u{fb04}' => Some("ffl"), + '\u{fb05}' | '\u{fb06}' => Some("st"), + _ => None, + } +} + +/// Expand typographic ligatures to their constituent ASCII letters. +/// +/// The PDF extraction path uses this directly instead of [`clean_text`]: +/// PDF Markdown must keep its line breaks, which `clean_text` collapses. +pub(crate) fn expand_ligatures(text: &str) -> String { + // `str::contains` takes a `Pattern`, and `RangeInclusive` only + // implements `Pattern` on nightly, so test chars explicitly. Membership + // comes from `ligature_expansion` so the fast path cannot drift from + // the expansion table. + if !text.chars().any(|c| ligature_expansion(c).is_some()) { + return text.to_string(); + } + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + match ligature_expansion(c) { + Some(expansion) => out.push_str(expansion), + None => out.push(c), + } + } + out +} + /// Collapse whitespace runs to single spaces. pub fn collapse_ws(text: &str) -> String { let mut out = String::with_capacity(text.len()); @@ -55,7 +89,7 @@ pub fn collapse_ws(text: &str) -> String { #[cfg(test)] mod tests { - use super::clean_text; + use super::{clean_text, expand_ligatures}; #[test] fn join_controls_preserved() { @@ -74,8 +108,20 @@ mod tests { assert_eq!(clean_text("a\u{fb00}ected"), "affected"); assert_eq!(clean_text("de\u{fb02}ect"), "deflect"); assert_eq!(clean_text("di\u{fb03}cult"), "difficult"); - assert_eq!(clean_text("waf\u{fb04}e"), "waffle"); + assert_eq!(clean_text("wa\u{fb04}e"), "waffle"); assert_eq!(clean_text("fa\u{fb05}"), "fast"); assert_eq!(clean_text("ju\u{fb06}"), "just"); } + + #[test] + fn expand_ligatures_pdf_path() { + assert_eq!( + expand_ligatures("\u{fb00}\u{fb01}\u{fb02}\u{fb03}\u{fb04}\u{fb05}\u{fb06}"), + "fffiflffifflstst" + ); + // Unlike clean_text, the PDF path must keep line breaks. + assert_eq!(expand_ligatures("a\nb\rc"), "a\nb\rc"); + // Fast path: text without ligatures is returned unchanged. + assert_eq!(expand_ligatures("plain text"), "plain text"); + } }