Skip to content
1        10        20        30        40        50        60        70        80        90        1001//! PDF in, Markdown out.2//!3//! This module owns everything about turning an uploaded PDF into Markdown;4//! the HTTP surface lives in the `http` submodule, the decision model is not5//! involved. It is deliberately free of web-framework types: the core is one6//! function over bytes, so it is unit-testable and the server can call it from7//! a blocking thread without borrowing anything actix-specific.8//!9//! ```no_run10//! use local_decisions::pdf::{self, Options};11//!12//! let bytes: &[u8] = &std::fs::read("document.pdf")?;13//! let extraction = pdf::extract_markdown(bytes, &Options::default())?;14//! println!("{}", extraction.markdown);15//! # Ok::<(), Box<dyn std::error::Error>>(())16//! ```17//!18//! The parse is backed by [`unpdf`](https://docs.rs/unpdf), a self-contained19//! Rust PDF parser — no external C libraries, no Python. Parsing is lenient by20//! default (a damaged document yields the pages that could be read); whatever21//! went missing along the way is reported in [`Extraction::incomplete`] and22//! [`Extraction::warnings`] rather than swallowed, so a page that silently23//! never arrived cannot be mistaken for a page that never existed.2425mod http;2627pub use http::route;2829use unpdf::render::{PageMarkerStyle, RenderOptions};3031/// Ceiling on an accepted upload, in bytes, unless the caller overrides32/// [`Options::max_bytes`]. Large enough for a real report, small enough that a33/// hostile upload cannot pin the CPU for a minute.34pub const DEFAULT_MAX_PDF_BYTES: usize = 20 * 1024 * 1024;3536/// How aggressively the extracted Markdown is cleaned up.37#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]38pub enum CleanupLevel {39    /// Keep the text as the parser produced it.40    None,41    /// Unicode normalisation only.42    Minimal,43    /// The unpdf default: collapse hyphenation and whitespace artefacts.44    #[default]45    Standard,46    /// For LLM consumption: also drop headers, footers and page furniture.47    Aggressive,48}4950/// Knobs for [`extract_markdown`]. Every field has a sensible default, so51/// `Options::default()` is what an endpoint should ship with.52#[derive(Debug, Clone)]53pub struct Options {54    /// Reject uploads larger than this before touching the parser.55    pub max_bytes: usize,56    /// Text cleanup applied to the extracted Markdown.57    pub cleanup: CleanupLevel,58    /// Insert `<!-- page N -->` markers at page boundaries.59    pub page_markers: bool,60    /// Prepend YAML frontmatter built from the document metadata.61    pub frontmatter: bool,62    /// Run the lossless Markdown shape-refinement pass.63    pub refine: bool,64}6566impl Default for Options {67    fn default() -> Self {68        Self {69            max_bytes: DEFAULT_MAX_PDF_BYTES,70            cleanup: CleanupLevel::default(),71            page_markers: false,72            frontmatter: false,73            refine: false,74        }75    }76}7778/// A successful conversion.79#[derive(Debug)]80pub struct Extraction {81    /// The document as Markdown, ready to hand to the decision API as `text`.82    pub markdown: String,83    /// Pages the document declares (1-indexed page numbers in the output).84    pub page_count: u32,85    /// Document title from the PDF metadata, when present.86    pub title: Option<String>,87    /// `true` when pages are known to be missing from the output.88    pub incomplete: bool,89    /// Everything a caller should know before trusting the text: unreadable90    /// content streams, suppressed text runs, damaged page sets.91    pub warnings: Vec<String>,92}9394/// Why [`extract_markdown`] refused a document.95///96/// [`status`] gives the HTTP status the error maps to, so the server layer97/// stays a thin `match`.98#[derive(Debug)]99pub enum PdfError {100    /// The upload exceeded [`Options::max_bytes`].101    TooLarge { size: usize, max: usize },102    /// The bytes do not start with the PDF magic number.103    NotPdf,104    /// The document is encrypted; encrypted PDFs are not accepted.105    Encrypted,106    /// The document is damaged beyond what lenient parsing can recover.107    Corrupted(String),108    /// The document parsed but produced no text — a pure scan, or every page109    /// blank. `scanned` distinguishes an image-only scan (OCR is not110    /// supported) from a document with no content at all.111    NoText { scanned: bool },112    /// Extraction failed in a way that is not the caller's fault.113    Internal(String),114}115116impl PdfError {117    /// HTTP status the server should answer with.118    #[must_use]119    pub fn status(&self) -> u16 {120        match self {121            Self::TooLarge { .. } => 413,122            Self::NotPdf | Self::Encrypted | Self::Corrupted(_) => 400,123            Self::NoText { .. } => 422,124            Self::Internal(_) => 500,125        }126    }127}128129impl std::fmt::Display for PdfError {130    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {131        match self {132            Self::TooLarge { size, max } => {133                write!(f, "the upload is {size} bytes; the limit is {max}")134            }135            Self::NotPdf => f.write_str("the upload is not a PDF"),136            Self::Encrypted => f.write_str("the PDF is encrypted, which is not supported"),137            Self::Corrupted(detail) => write!(f, "the PDF is damaged: {detail}"),138            Self::NoText { scanned: true } => {139                f.write_str("the PDF has no text layer (it is a scan); OCR is not supported")140            }141            Self::NoText { scanned: false } => f.write_str("the PDF contains no extractable text"),142            Self::Internal(detail) => write!(f, "PDF extraction failed: {detail}"),143        }144    }145}146147impl std::error::Error for PdfError {}148149/// Convert PDF bytes to Markdown.150///151/// # Errors152/// Returns a [`PdfError`] when the upload is not an acceptable PDF; see the153/// variant list for the reasons and [`PdfError::status`] for how they map to154/// HTTP. Parsing is CPU-bound: call this off the request thread (the actix155/// wrapper in [`http`] already does, via `web::block`).156pub fn extract_markdown(bytes: &[u8], options: &Options) -> Result<Extraction, PdfError> {157    if bytes.len() > options.max_bytes {158        return Err(PdfError::TooLarge {159            size: bytes.len(),160            max: options.max_bytes,161        });162    }163    // A clean rejection for garbage before the parser gets a say.164    if unpdf::detect_format_from_bytes(bytes).is_err() {165        return Err(PdfError::NotPdf);166    }167168    let doc = unpdf::parse_bytes(bytes).map_err(|error| map_parse_error(&error))?;169    if doc.page_count() == 0 {170        return Err(PdfError::Corrupted(171            "the document contains no readable pages".to_owned(),172        ));173    }174175    let mut render = RenderOptions::new()176        .with_frontmatter(options.frontmatter)177        .with_page_markers(if options.page_markers {178            PageMarkerStyle::Comment179        } else {180            PageMarkerStyle::None181        });182    render.cleanup = match options.cleanup {183        CleanupLevel::None => None,184        CleanupLevel::Minimal => Some(unpdf::render::CleanupOptions::from_preset(185            unpdf::render::CleanupPreset::Minimal,186        )),187        CleanupLevel::Standard => Some(unpdf::render::CleanupOptions::standard()),188        CleanupLevel::Aggressive => Some(unpdf::render::CleanupOptions::from_preset(189            unpdf::render::CleanupPreset::Aggressive,190        )),191    };192    if options.refine {193        render = render.with_refine();194    }195196    let markdown =197        unpdf::render::to_markdown(&doc, &render).map_err(|e| PdfError::Internal(e.to_string()))?;198    if markdown.trim().is_empty() {199        return Err(PdfError::NoText {200            scanned: doc.extraction_quality.is_scan_pdf,201        });202    }203204    let quality = &doc.extraction_quality;205    let incomplete = quality.pages_incomplete;206    let mut warnings = Vec::new();207    if incomplete {208        warnings.push(match quality.declared_page_count {209            Some(declared) => format!(210                "extraction is incomplete: {declared} pages declared, {} extracted",211                doc.page_count()212            ),213            None => "extraction is incomplete: the declared page count was unreadable".to_owned(),214        });215    }216    if quality.suppressed_text_runs > 0 {217        warnings.push(format!(218            "{} text run(s) could not be decoded and are missing from the output",219            quality.suppressed_text_runs220        ));221    }222    if quality.undecodable_content_streams > 0 {223        warnings.push(format!(224            "{} content stream(s) could not be decoded and are missing from the output",225            quality.undecodable_content_streams226        ));227    }228    if let Some(warning) = quality.warning_message() {229        warnings.push(warning);230    }231232    Ok(Extraction {233        markdown,234        page_count: doc.page_count(),235        title: doc.metadata.title.clone(),236        incomplete,237        warnings,238    })239}240241/// Sort an `unpdf` error into the caller-facing variants.242fn map_parse_error(error: &unpdf::Error) -> PdfError {243    match error.kind() {244        unpdf::ErrorKind::Encrypted | unpdf::ErrorKind::InvalidPassword => PdfError::Encrypted,245        unpdf::ErrorKind::UnknownFormat => PdfError::NotPdf,246        unpdf::ErrorKind::Corrupted247        | unpdf::ErrorKind::PdfParse248        | unpdf::ErrorKind::UnsupportedVersion249        | unpdf::ErrorKind::MissingObject250        | unpdf::ErrorKind::PageOutOfRange251        | unpdf::ErrorKind::InvalidPageRange => PdfError::Corrupted(error.to_string()),252        _ => PdfError::Internal(error.to_string()),253    }254}255256#[cfg(test)]257pub(crate) mod tests {258    use super::*;259260    /// Build a valid single- or multi-page PDF with computed xref offsets.261    /// Each entry becomes one page with one line of Helvetica text.262    pub(crate) fn tiny_pdf(pages: &[&str]) -> Vec<u8> {263        let page_count = pages.len();264        let font_num = 3 + 2 * page_count;265        let mut objects: Vec<Vec<u8>> = Vec::new();266        objects.push(b"<< /Type /Catalog /Pages 2 0 R >>".to_vec());267        let kids = (0..page_count)268            .map(|i| format!("{} 0 R", 3 + 2 * i))269            .collect::<Vec<_>>()270            .join(" ");271        objects.push(format!("<< /Type /Pages /Kids [{kids}] /Count {page_count} >>").into_bytes());272        for (i, text) in pages.iter().enumerate() {273            objects.push(274                format!(275                    "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] \276                     /Resources << /Font << /F1 {font_num} 0 R >> >> /Contents {} 0 R >>",277                    4 + 2 * i278                )279                .into_bytes(),280            );281            let stream = format!("BT /F1 24 Tf 72 720 Td ({text}) Tj ET");282            objects.push(283                format!(284                    "<< /Length {} >>\nstream\n{stream}\nendstream",285                    stream.len()286                )287                .into_bytes(),288            );289        }290        objects.push(291            b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>"292                .to_vec(),293        );294295        let mut out = b"%PDF-1.4\n".to_vec();296        let mut offsets = Vec::new();297        for (index, object) in objects.iter().enumerate() {298            offsets.push(out.len());299            out.extend_from_slice(format!("{} 0 obj\n", index + 1).as_bytes());300            out.extend_from_slice(object);301            out.extend_from_slice(b"\nendobj\n");302        }303        let xref = out.len();304        out.extend_from_slice(format!("xref\n0 {}\n", objects.len() + 1).as_bytes());305        out.extend_from_slice(b"0000000000 65535 f \n");306        for offset in offsets {307            out.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes());308        }309        out.extend_from_slice(310            format!(311                "trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n",312                objects.len() + 1313            )314            .as_bytes(),315        );316        out317    }318319    fn extract(bytes: &[u8]) -> Result<Extraction, PdfError> {320        extract_markdown(bytes, &Options::default())321    }322323    #[test]324    fn extracts_text_from_a_tiny_pdf() {325        let extraction = extract(&tiny_pdf(&["Hello decision engine"])).expect("extracts");326        assert_eq!(extraction.page_count, 1);327        assert!(extraction.markdown.contains("Hello decision engine"));328        assert!(!extraction.incomplete);329        assert!(extraction.warnings.is_empty());330    }331332    #[test]333    fn page_markers_and_counts_survive_multiple_pages() {334        let options = Options {335            page_markers: true,336            ..Options::default()337        };338        let extraction = extract_markdown(&tiny_pdf(&["first page", "second page"]), &options)339            .expect("extracts");340        assert_eq!(extraction.page_count, 2);341        assert!(extraction.markdown.contains("<!-- page 1 -->"));342        assert!(extraction.markdown.contains("<!-- page 2 -->"));343        assert!(extraction.markdown.contains("second page"));344    }345346    #[test]347    fn rejects_non_pdf_bytes() {348        let error = extract(b"this is not a pdf at all").expect_err("rejected");349        assert!(matches!(error, PdfError::NotPdf));350        assert_eq!(error.status(), 400);351    }352353    #[test]354    fn rejects_oversized_pdfs() {355        let options = Options {356            max_bytes: 16,357            ..Options::default()358        };359        let bytes = tiny_pdf(&["Hello decision engine"]);360        assert!(bytes.len() > 16);361        let error = extract_markdown(&bytes, &options).expect_err("rejected");362        assert!(matches!(error, PdfError::TooLarge { .. }));363        assert_eq!(error.status(), 413);364    }365366    #[test]367    fn rejects_a_textless_document() {368        let bytes = tiny_pdf(&[""]);369        let error = extract(&bytes).expect_err("rejected");370        assert!(matches!(error, PdfError::NoText { scanned: false }));371        assert_eq!(error.status(), 422);372    }373}