LINE1 10 20 30 40 50 60 70 80 90 1001//! PDF in, Markdown out.2//!3//! This module owns everything about turning an uploaded PDF into Markdown;4//! the HTTP surface lives in the `http` submodule, the decision model is not5//! involved. It is deliberately free of web-framework types: the core is one6//! function over bytes, so it is unit-testable and the server can call it from7//! a blocking thread without borrowing anything actix-specific.8//!9//! ```no_run10//! use local_decisions::pdf::{self, Options};11//!12//! let bytes: &[u8] = &std::fs::read("document.pdf")?;13//! let extraction = pdf::extract_markdown(bytes, &Options::default())?;14//! println!("{}", extraction.markdown);15//! # Ok::<(), Box<dyn std::error::Error>>(())16//! ```17//!18//! The parse is backed by [`unpdf`](https://docs.rs/unpdf), a self-contained19//! Rust PDF parser — no external C libraries, no Python. Parsing is lenient by20//! default (a damaged document yields the pages that could be read); whatever21//! went missing along the way is reported in [`Extraction::incomplete`] and22//! [`Extraction::warnings`] rather than swallowed, so a page that silently23//! never arrived cannot be mistaken for a page that never existed.2425mod http;2627pubuse http::route;2829use unpdf::render::{PageMarkerStyle, RenderOptions};3031/// Ceiling on an accepted upload, in bytes, unless the caller overrides32/// [`Options::max_bytes`]. Large enough for a real report, small enough that a33/// hostile upload cannot pin the CPU for a minute.34pubconstDEFAULT_MAX_PDF_BYTES: usize = 20 * 1024 * 1024;3536/// How aggressively the extracted Markdown is cleaned up.37#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]38pubenumCleanupLevel {39/// Keep the text as the parser produced it.40None,41/// Unicode normalisation only.42Minimal,43/// The unpdf default: collapse hyphenation and whitespace artefacts.44#[default]45Standard,46/// For LLM consumption: also drop headers, footers and page furniture.47Aggressive,48}4950/// Knobs for [`extract_markdown`]. Every field has a sensible default, so51/// `Options::default()` is what an endpoint should ship with.52#[derive(Debug, Clone)]53pubstructOptions {54/// Reject uploads larger than this before touching the parser.55pub max_bytes: usize,56/// Text cleanup applied to the extracted Markdown.57pub cleanup: CleanupLevel,58/// Insert `<!-- page N -->` markers at page boundaries.59pub page_markers: bool,60/// Prepend YAML frontmatter built from the document metadata.61pub frontmatter: bool,62/// Run the lossless Markdown shape-refinement pass.63pub refine: bool,64}6566implDefaultforOptions {67fndefault() -> Self {68Self {69 max_bytes: DEFAULT_MAX_PDF_BYTES,70 cleanup: CleanupLevel::default(),71 page_markers: false,72 frontmatter: false,73 refine: false,74 }75 }76}7778/// A successful conversion.79#[derive(Debug)]80pubstructExtraction {81/// The document as Markdown, ready to hand to the decision API as `text`.82pub markdown: String,83/// Pages the document declares (1-indexed page numbers in the output).84pub page_count: u32,85/// Document title from the PDF metadata, when present.86pub title: Option<String>,87/// `true` when pages are known to be missing from the output.88pub incomplete: bool,89/// Everything a caller should know before trusting the text: unreadable90/// content streams, suppressed text runs, damaged page sets.91pub warnings: Vec<String>,92}9394/// Why [`extract_markdown`] refused a document.95///96/// [`status`] gives the HTTP status the error maps to, so the server layer97/// stays a thin `match`.98#[derive(Debug)]99pubenumPdfError {100/// The upload exceeded [`Options::max_bytes`].101TooLarge { size: usize, max: usize },102/// The bytes do not start with the PDF magic number.103NotPdf,104/// The document is encrypted; encrypted PDFs are not accepted.105Encrypted,106/// The document is damaged beyond what lenient parsing can recover.107Corrupted(String),108/// The document parsed but produced no text — a pure scan, or every page109/// blank. `scanned` distinguishes an image-only scan (OCR is not110/// supported) from a document with no content at all.111NoText { scanned: bool },112/// Extraction failed in a way that is not the caller's fault.113Internal(String),114}115116implPdfError {117/// HTTP status the server should answer with.118#[must_use]119pubfnstatus(&self) -> u16 {120matchself {121Self::TooLarge { .. } => 413,122Self::NotPdf | Self::Encrypted | Self::Corrupted(_) => 400,123Self::NoText { .. } => 422,124Self::Internal(_) => 500,125 }126 }127}128129impl std::fmt::DisplayforPdfError {130fnfmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {131matchself {132Self::TooLarge { size, max } => {133write!(f, "the upload is {size} bytes; the limit is {max}")134 }135Self::NotPdf => f.write_str("the upload is not a PDF"),136Self::Encrypted => f.write_str("the PDF is encrypted, which is not supported"),137Self::Corrupted(detail) => write!(f, "the PDF is damaged: {detail}"),138Self::NoText { scanned: true } => {139 f.write_str("the PDF has no text layer (it is a scan); OCR is not supported")140 }141Self::NoText { scanned: false } => f.write_str("the PDF contains no extractable text"),142Self::Internal(detail) => write!(f, "PDF extraction failed: {detail}"),143 }144 }145}146147impl std::error::ErrorforPdfError {}148149/// Convert PDF bytes to Markdown.150///151/// # Errors152/// Returns a [`PdfError`] when the upload is not an acceptable PDF; see the153/// variant list for the reasons and [`PdfError::status`] for how they map to154/// HTTP. Parsing is CPU-bound: call this off the request thread (the actix155/// wrapper in [`http`] already does, via `web::block`).156pubfnextract_markdown(bytes: &[u8], options: &Options) -> Result<Extraction, PdfError> {157if bytes.len() > options.max_bytes {158returnErr(PdfError::TooLarge {159 size: bytes.len(),160 max: options.max_bytes,161 });162 }163// A clean rejection for garbage before the parser gets a say.164if unpdf::detect_format_from_bytes(bytes).is_err() {165returnErr(PdfError::NotPdf);166 }167168let doc = unpdf::parse_bytes(bytes).map_err(|error| map_parse_error(&error))?;169if doc.page_count() == 0 {170returnErr(PdfError::Corrupted(171"the document contains no readable pages".to_owned(),172 ));173 }174175letmut render = RenderOptions::new()176 .with_frontmatter(options.frontmatter)177 .with_page_markers(if options.page_markers {178PageMarkerStyle::Comment179 } else {180PageMarkerStyle::None181 });182 render.cleanup = match options.cleanup {183CleanupLevel::None => None,184CleanupLevel::Minimal => Some(unpdf::render::CleanupOptions::from_preset(185 unpdf::render::CleanupPreset::Minimal,186 )),187CleanupLevel::Standard => Some(unpdf::render::CleanupOptions::standard()),188CleanupLevel::Aggressive => Some(unpdf::render::CleanupOptions::from_preset(189 unpdf::render::CleanupPreset::Aggressive,190 )),191 };192if options.refine {193 render = render.with_refine();194 }195196let markdown =197 unpdf::render::to_markdown(&doc, &render).map_err(|e| PdfError::Internal(e.to_string()))?;198if markdown.trim().is_empty() {199returnErr(PdfError::NoText {200 scanned: doc.extraction_quality.is_scan_pdf,201 });202 }203204let quality = &doc.extraction_quality;205let incomplete = quality.pages_incomplete;206letmut warnings = Vec::new();207if incomplete {208 warnings.push(match quality.declared_page_count {209Some(declared) => format!(210"extraction is incomplete: {declared} pages declared, {} extracted",211 doc.page_count()212 ),213None => "extraction is incomplete: the declared page count was unreadable".to_owned(),214 });215 }216if quality.suppressed_text_runs > 0 {217 warnings.push(format!(218"{} text run(s) could not be decoded and are missing from the output",219 quality.suppressed_text_runs220 ));221 }222if quality.undecodable_content_streams > 0 {223 warnings.push(format!(224"{} content stream(s) could not be decoded and are missing from the output",225 quality.undecodable_content_streams226 ));227 }228ifletSome(warning) = quality.warning_message() {229 warnings.push(warning);230 }231232Ok(Extraction {233 markdown,234 page_count: doc.page_count(),235 title: doc.metadata.title.clone(),236 incomplete,237 warnings,238 })239}240241/// Sort an `unpdf` error into the caller-facing variants.242fnmap_parse_error(error: &unpdf::Error) -> PdfError {243match error.kind() {244 unpdf::ErrorKind::Encrypted | unpdf::ErrorKind::InvalidPassword => PdfError::Encrypted,245 unpdf::ErrorKind::UnknownFormat => PdfError::NotPdf,246 unpdf::ErrorKind::Corrupted247 | unpdf::ErrorKind::PdfParse248 | unpdf::ErrorKind::UnsupportedVersion249 | unpdf::ErrorKind::MissingObject250 | unpdf::ErrorKind::PageOutOfRange251 | unpdf::ErrorKind::InvalidPageRange => PdfError::Corrupted(error.to_string()),252 _ => PdfError::Internal(error.to_string()),253 }254}255256#[cfg(test)]257pub(crate) mod tests {258usesuper::*;259260/// Build a valid single- or multi-page PDF with computed xref offsets.261/// Each entry becomes one page with one line of Helvetica text.262pub(crate) fntiny_pdf(pages: &[&str]) -> Vec<u8> {263let page_count = pages.len();264let font_num = 3 + 2 * page_count;265letmut objects: Vec<Vec<u8>> = Vec::new();266 objects.push(b"<< /Type /Catalog /Pages 2 0 R >>".to_vec());267let kids = (0..page_count)268 .map(|i| format!("{} 0 R", 3 + 2 * i))269 .collect::<Vec<_>>()270 .join(" ");271 objects.push(format!("<< /Type /Pages /Kids [{kids}] /Count {page_count} >>").into_bytes());272for (i, text) in pages.iter().enumerate() {273 objects.push(274format!(275 "<< /Type /Page /Parent20R /MediaBox [00612792] \276 /Resources << /Font << /F1 {font_num} 0R >> >> /Contents {} 0R >>",277 4 + 2 * i278 )279 .into_bytes(),280 );281 let stream = format!("BT /F124Tf72720Td ({text}) TjET");282 objects.push(283 format!(284 "<< /Length {} >>\nstream\n{stream}\nendstream",285 stream.len()286 )287 .into_bytes(),288 );289 }290 objects.push(291 b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>"292 .to_vec(),293 );294295 let mut out = b"%PDF-1.4\n".to_vec();296 let mut offsets = Vec::new();297 for (index, object) in objects.iter().enumerate() {298 offsets.push(out.len());299 out.extend_from_slice(format!("{} 0 obj\n", index + 1).as_bytes());300 out.extend_from_slice(object);301 out.extend_from_slice(b"\nendobj\n");302 }303 let xref = out.len();304 out.extend_from_slice(format!("xref\n0 {}\n", objects.len() + 1).as_bytes());305 out.extend_from_slice(b"000000000065535 f \n");306 for offset in offsets {307 out.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());308 }309 out.extend_from_slice(310 format!(311 "trailer\n<< /Size {} /Root10R >>\nstartxref\n{xref}\n%%EOF\n",312 objects.len() + 1313 )314 .as_bytes(),315 );316 out317 }318319 fn extract(bytes: &[u8]) -> Result<Extraction, PdfError> {320 extract_markdown(bytes, &Options::default())321 }322323 #[test]324 fn extracts_text_from_a_tiny_pdf() {325 let extraction = extract(&tiny_pdf(&["Hello decision engine"])).expect("extracts");326 assert_eq!(extraction.page_count, 1);327 assert!(extraction.markdown.contains("Hello decision engine"));328 assert!(!extraction.incomplete);329 assert!(extraction.warnings.is_empty());330 }331332 #[test]333 fn page_markers_and_counts_survive_multiple_pages() {334 let options = Options {335 page_markers: true,336 ..Options::default()337 };338 let extraction = extract_markdown(&tiny_pdf(&["first page", "second page"]), &options)339 .expect("extracts");340 assert_eq!(extraction.page_count, 2);341 assert!(extraction.markdown.contains("<!-- page 1 -->"));342 assert!(extraction.markdown.contains("<!-- page 2 -->"));343 assert!(extraction.markdown.contains("second page"));344 }345346 #[test]347 fn rejects_non_pdf_bytes() {348 let error = extract(b"this is not a pdf at all").expect_err("rejected");349 assert!(matches!(error, PdfError::NotPdf));350 assert_eq!(error.status(), 400);351 }352353 #[test]354 fn rejects_oversized_pdfs() {355 let options = Options {356 max_bytes: 16,357 ..Options::default()358 };359 let bytes = tiny_pdf(&["Hello decision engine"]);360 assert!(bytes.len() > 16);361 let error = extract_markdown(&bytes, &options).expect_err("rejected");362 assert!(matches!(error, PdfError::TooLarge { .. }));363 assert_eq!(error.status(), 413);364 }365366 #[test]367 fn rejects_a_textless_document() {368 let bytes = tiny_pdf(&[""]);369 let error = extract(&bytes).expect_err("rejected");370assert!(matches!(error, PdfError::NoText { scanned: false }));371assert_eq!(error.status(), 422);372 }373}