CodeSource
/
Readur
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(pdf): implement ocrmypdf to extract text from PDFs

This commit is contained in:
perf3ct 2025-07-01 00:56:48 +00:00
parent 59e80a1b92
commit f7018575d8
No known key found for this signature in database
GPG Key ID: 569C4EEC436F5232
15 changed files with 1222 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -20,6 +20,7 @@ RUN apt-get update && apt-get install -y \
libclang-dev \ libclang-dev \
clang \ clang \
poppler-utils \ poppler-utils \
ocrmypdf \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
WORKDIR /app WORKDIR /app
@ -37,6 +38,7 @@ RUN apt-get update && apt-get install -y \
tesseract-ocr-eng \ tesseract-ocr-eng \
ca-certificates \ ca-certificates \
poppler-utils \ poppler-utils \
ocrmypdf \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
WORKDIR /app WORKDIR /app

162
create_test_pdfs.py Normal file
View File

@ -0,0 +1,162 @@
#!/usr/bin/env python3
"""
Create proper test PDFs for debugging OCR word counting issues.
"""
try:
from reportlab.pdfgen import canvas
from reportlab.lib.pagesizes import letter
import os
except ImportError:
print("reportlab not installed. Trying alternative method...")
# Alternative: create simple text files for testing
import os
def create_simple_test_files():
"""Create simple text files as a fallback"""
test_dir = "tests/test_pdfs"
os.makedirs(test_dir, exist_ok=True)
# Test cases that would be similar to PDF extraction results
test_cases = [
("normal_spacing.txt", "This is a normal document with proper word spacing and punctuation."),
("soclogix_sample.txt", "SOCLogix Non-Disclosure Agreement\nThis agreement is entered into between SOCLogix and the recipient for the purpose of protecting confidential information."),
("multiline_text.txt", "Line one with several words\nLine two with more content\nLine three continues the pattern\nFinal line ends the document"),
("mixed_content.txt", "Document with numbers 123 and symbols @#$ mixed with normal text."),
("special_chars.txt", "Text with special characters: café naïve résumé — and 'quotes' • bullets"),
]
for filename, content in test_cases:
with open(f"{test_dir}/{filename}", "w", encoding="utf-8") as f:
f.write(content)
print("Created simple text files for testing")
return True
if not create_simple_test_files():
exit(1)
exit(0)
def create_test_pdfs():
"""Create proper test PDFs using reportlab"""
test_dir = "tests/test_pdfs"
os.makedirs(test_dir, exist_ok=True)
# Test case 1: Normal spacing (like SOCLogix NDA)
pdf_path = f"{test_dir}/soclogix_nda_realistic.pdf"
c = canvas.Canvas(pdf_path, pagesize=letter)
width, height = letter
# Add text with normal spacing
c.setFont("Helvetica", 12)
y_position = height - 100
lines = [
"SOCLogix Non-Disclosure Agreement",
"",
"This agreement is entered into between SOCLogix and the recipient",
"for the purpose of protecting confidential information.",
"",
"The recipient agrees to maintain strict confidentiality",
"regarding all proprietary information disclosed.",
"",
"This includes but is not limited to technical specifications,",
"business plans, customer lists, and financial data.",
"",
"Any breach of this agreement may result in legal action.",
"The agreement remains in effect for a period of five years.",
]
for line in lines:
if line: # Skip empty lines for positioning
c.drawString(72, y_position, line)
y_position -= 20
c.save()
print(f"Created: {pdf_path}")
# Test case 2: Multi-page document
pdf_path = f"{test_dir}/multipage_realistic.pdf"
c = canvas.Canvas(pdf_path, pagesize=letter)
# Page 1
c.setFont("Helvetica", 12)
y_position = height - 100
page1_lines = [
"Page 1: Document with Multiple Pages",
"",
"This is the first page of a multi-page document.",
"It contains multiple sentences with proper spacing.",
"Each line should be counted as separate words.",
"Word boundaries are clearly defined with spaces.",
"",
"Numbers like 123, 456, and 789 should also count.",
"Punctuation marks help separate thoughts.",
"Total words on this page should be easily counted.",
]
for line in page1_lines:
if line:
c.drawString(72, y_position, line)
y_position -= 20
# Start new page
c.showPage()
y_position = height - 100
page2_lines = [
"Page 2: Continuing from Previous Page",
"",
"This page also has normal text formatting.",
"Word counting should work correctly here too.",
"Mixed content: ABC123 def456 GHI789 works fine.",
"",
"Special characters like café, naïve, and résumé",
"should also be handled properly by the extraction.",
"",
"End of document with proper word boundaries.",
]
for line in page2_lines:
if line:
c.drawString(72, y_position, line)
y_position -= 20
c.save()
print(f"Created: {pdf_path}")
# Test case 3: Document with problematic patterns
pdf_path = f"{test_dir}/edge_cases_realistic.pdf"
c = canvas.Canvas(pdf_path, pagesize=letter)
c.setFont("Helvetica", 12)
y_position = height - 100
edge_case_lines = [
"Edge Cases for Word Counting",
"",
"Normal text with proper spacing works fine.",
"TextWithoutSpacesButCamelCase should be detected.",
"ALLCAPSTEXT might be problematic.",
"mixed123CASE456text789 has transitions.",
"",
"Punctuation!!! should not count as words.",
"But text-with-hyphens should count properly.",
"Email@example.com and URLs http://test.com too.",
"",
"End with normal text to verify counting.",
]
for line in edge_case_lines:
if line:
c.drawString(72, y_position, line)
y_position -= 20
c.save()
print(f"Created: {pdf_path}")
print("\nAll test PDFs created successfully!")
return True
if __name__ == "__main__":
create_test_pdfs()

244
src/ocr/enhanced.rs
View File

@ -791,7 +791,7 @@ impl EnhancedOcrService {
/// Extract text from PDF with size and time limits /// Extract text from PDF with size and time limits
#[cfg(feature = "ocr")] #[cfg(feature = "ocr")]
pub async fn extract_text_from_pdf(&self, file_path: &str, _settings: &Settings) -> Result<OcrResult> { pub async fn extract_text_from_pdf(&self, file_path: &str, settings: &Settings) -> Result<OcrResult> {
let start_time = std::time::Instant::now(); let start_time = std::time::Instant::now();
info!("Extracting text from PDF: {}", file_path); info!("Extracting text from PDF: {}", file_path);
@ -888,14 +888,188 @@ impl EnhancedOcrService {
trimmed_text.chars().take(200).collect::<String>() trimmed_text.chars().take(200).collect::<String>()
); );
// Smart detection: assess if text extraction quality is good enough
if self.is_text_extraction_quality_sufficient(&trimmed_text, word_count, file_size) {
info!("PDF text extraction successful for '{}', using extracted text", file_path);
Ok(OcrResult { Ok(OcrResult {
text: trimmed_text, text: trimmed_text,
confidence: 95.0, // PDF text extraction is generally high confidence confidence: 95.0, // PDF text extraction is generally high confidence
processing_time_ms: processing_time, processing_time_ms: processing_time,
word_count, word_count,
preprocessing_applied: vec!["PDF text extraction".to_string()], preprocessing_applied: vec!["PDF text extraction".to_string()],
processed_image_path: None, // No image processing for PDF text extraction processed_image_path: None,
}) })
} else {
info!("PDF text extraction insufficient for '{}' ({} words), falling back to OCR", file_path, word_count);
// Fall back to OCR using ocrmypdf
self.extract_text_from_pdf_with_ocr(file_path, settings, start_time).await
}
}
/// Assess if text extraction quality is sufficient or if OCR fallback is needed
#[cfg(feature = "ocr")]
fn is_text_extraction_quality_sufficient(&self, text: &str, word_count: usize, file_size: u64) -> bool {
// If we got no words at all, definitely need OCR
if word_count == 0 {
return false;
}
// For very small files, low word count might be normal
if file_size < 50_000 && word_count >= 1 {
return true;
}
// Calculate word density (words per KB)
let file_size_kb = (file_size as f64) / 1024.0;
let word_density = (word_count as f64) / file_size_kb;
// Reasonable thresholds based on typical PDF content:
// - Text-based PDFs typically have 50-200 words per KB
// - Below 5 words per KB suggests mostly images/scanned content
const MIN_WORD_DENSITY: f64 = 5.0;
const MIN_WORDS_FOR_LARGE_FILES: usize = 10;
if word_density < MIN_WORD_DENSITY && word_count < MIN_WORDS_FOR_LARGE_FILES {
debug!("PDF appears to be image-based: {} words in {:.1} KB (density: {:.2} words/KB)",
word_count, file_size_kb, word_density);
return false;
}
// Additional check: if text is mostly non-alphanumeric, might be extraction artifacts
let alphanumeric_chars = text.chars().filter(|c| c.is_alphanumeric()).count();
let alphanumeric_ratio = if text.len() > 0 {
(alphanumeric_chars as f64) / (text.len() as f64)
} else {
0.0
};
// If less than 30% alphanumeric content, likely poor extraction
if alphanumeric_ratio < 0.3 {
debug!("PDF text has low alphanumeric content: {:.1}% ({} of {} chars)",
alphanumeric_ratio * 100.0, alphanumeric_chars, text.len());
return false;
}
debug!("PDF text extraction quality sufficient: {} words, {:.2} words/KB, {:.1}% alphanumeric",
word_count, word_density, alphanumeric_ratio * 100.0);
true
}
/// Extract text from PDF using OCR (ocrmypdf) for image-based or poor-quality PDFs
#[cfg(feature = "ocr")]
async fn extract_text_from_pdf_with_ocr(&self, file_path: &str, settings: &Settings, start_time: std::time::Instant) -> Result<OcrResult> {
info!("Starting OCR extraction for PDF: {}", file_path);
// Check if ocrmypdf is available
if !self.is_ocrmypdf_available().await {
return Err(anyhow!(
"ocrmypdf is not available on this system. To extract text from image-based PDFs like '{}', please install ocrmypdf. \
On Ubuntu/Debian: 'apt-get install ocrmypdf'. \
On macOS: 'brew install ocrmypdf'. \
Alternatively, convert the PDF to images and upload those instead.",
file_path
));
}
// Generate temporary file path for OCR'd PDF
let temp_ocr_filename = format!("ocr_{}_{}.pdf",
std::process::id(),
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)?.as_millis()
);
let temp_ocr_path = format!("{}/{}", self.temp_dir, temp_ocr_filename);
// Run ocrmypdf to create searchable PDF
let ocrmypdf_result = tokio::time::timeout(
std::time::Duration::from_secs(300), // 5 minute timeout for OCR
tokio::task::spawn_blocking({
let file_path = file_path.to_string();
let temp_ocr_path = temp_ocr_path.clone();
move || {
std::process::Command::new("ocrmypdf")
.arg("--force-ocr") // OCR even if text is detected
.arg("-O2") // Optimize level 2 (balanced quality/speed)
.arg("--deskew") // Correct skewed pages
.arg("--clean") // Clean up artifacts
.arg("--language")
.arg("eng") // English language
.arg(&file_path)
.arg(&temp_ocr_path)
.output()
}
})
).await;
let ocrmypdf_output = match ocrmypdf_result {
Ok(Ok(output)) => output?,
Ok(Err(e)) => return Err(anyhow!("Failed to join ocrmypdf task: {}", e)),
Err(_) => return Err(anyhow!("ocrmypdf timed out after 5 minutes for file '{}'", file_path)),
};
if !ocrmypdf_output.status.success() {
let stderr = String::from_utf8_lossy(&ocrmypdf_output.stderr);
let stdout = String::from_utf8_lossy(&ocrmypdf_output.stdout);
return Err(anyhow!(
"ocrmypdf failed for '{}': Exit code {}\nStderr: {}\nStdout: {}",
file_path, ocrmypdf_output.status.code().unwrap_or(-1), stderr, stdout
));
}
// Extract text from the OCR'd PDF
let ocr_text_result = tokio::task::spawn_blocking({
let temp_ocr_path = temp_ocr_path.clone();
move || -> Result<String> {
let bytes = std::fs::read(&temp_ocr_path)?;
let text = pdf_extract::extract_text_from_mem(&bytes)?;
Ok(text.trim().to_string())
}
}).await??;
// Clean up temporary file
let _ = tokio::fs::remove_file(&temp_ocr_path).await;
let processing_time = start_time.elapsed().as_millis() as u64;
let word_count = self.count_words_safely(&ocr_text_result);
info!("OCR extraction completed for '{}': {} words in {}ms",
file_path, word_count, processing_time);
Ok(OcrResult {
text: ocr_text_result,
confidence: 85.0, // OCR is generally lower confidence than direct text extraction
processing_time_ms: processing_time,
word_count,
preprocessing_applied: vec!["OCR via ocrmypdf".to_string()],
processed_image_path: None,
})
}
/// Check if ocrmypdf is available on the system
#[cfg(feature = "ocr")]
async fn is_ocrmypdf_available(&self) -> bool {
match tokio::process::Command::new("ocrmypdf")
.arg("--version")
.output()
.await
{
Ok(output) => output.status.success(),
Err(_) => false,
}
}
#[cfg(not(feature = "ocr"))]
fn is_text_extraction_quality_sufficient(&self, _text: &str, _word_count: usize, _file_size: u64) -> bool {
// When OCR is disabled, always accept text extraction results
true
}
#[cfg(not(feature = "ocr"))]
async fn is_ocrmypdf_available(&self) -> bool {
false // OCR feature not enabled
}
#[cfg(not(feature = "ocr"))]
async fn extract_text_from_pdf_with_ocr(&self, file_path: &str, _settings: &Settings, _start_time: std::time::Instant) -> Result<OcrResult> {
Err(anyhow::anyhow!("OCR feature not enabled - cannot process image-based PDF: {}", file_path))
} }
/// Resolve file path to actual location, handling both old and new directory structures /// Resolve file path to actual location, handling both old and new directory structures
@ -988,7 +1162,7 @@ impl EnhancedOcrService {
/// Safely count words to prevent overflow on very large texts /// Safely count words to prevent overflow on very large texts
#[cfg(feature = "ocr")] #[cfg(feature = "ocr")]
fn count_words_safely(&self, text: &str) -> usize { pub fn count_words_safely(&self, text: &str) -> usize {
// For very large texts, sample to estimate word count to prevent overflow // For very large texts, sample to estimate word count to prevent overflow
if text.len() > 1_000_000 { // > 1MB of text if text.len() > 1_000_000 { // > 1MB of text
// Sample first 100KB and extrapolate // Sample first 100KB and extrapolate
@ -1008,31 +1182,51 @@ impl EnhancedOcrService {
fn count_words_in_text(&self, text: &str) -> usize { fn count_words_in_text(&self, text: &str) -> usize {
let whitespace_words = text.split_whitespace().count(); let whitespace_words = text.split_whitespace().count();
// If no whitespace-separated words found but text exists, try alternative word detection // If we have exactly 1 "word" but it's very long (likely continuous text), try enhanced detection
if whitespace_words == 0 && !text.trim().is_empty() { // OR if we have no whitespace words but text exists
// For PDFs that extract as continuous text, estimate words based on character patterns let is_continuous_text = whitespace_words == 1 && text.len() > 15; // 15+ chars suggests it might be continuous
// Look for transitions from letters to non-letters as potential word boundaries let is_no_words = whitespace_words == 0 && !text.trim().is_empty();
let mut word_count = 0;
let mut in_word = false;
for c in text.chars() { if is_continuous_text || is_no_words {
if c.is_alphabetic() { // Count total alphanumeric characters first
if !in_word {
word_count += 1;
in_word = true;
}
} else {
in_word = false;
}
}
// If still no words found but we have alphanumeric content,
// estimate based on reasonable word length (assume ~5 chars per word)
if word_count == 0 {
let alphanumeric_chars = text.chars().filter(|c| c.is_alphanumeric()).count(); let alphanumeric_chars = text.chars().filter(|c| c.is_alphanumeric()).count();
if alphanumeric_chars > 0 {
word_count = (alphanumeric_chars / 5).max(1); // If no alphanumeric content, it's pure punctuation/symbols
if alphanumeric_chars == 0 {
return 0;
} }
// For continuous text, look for word boundaries using multiple strategies
let mut word_count = 0;
// Strategy 1: Count transitions from lowercase to uppercase (camelCase detection)
let chars: Vec<char> = text.chars().collect();
let mut camel_transitions = 0;
for i in 1..chars.len() {
let prev_char = chars[i-1];
let curr_char = chars[i];
// Count transitions from lowercase letter to uppercase letter
if prev_char.is_lowercase() && curr_char.is_uppercase() {
camel_transitions += 1;
}
// Count transitions from letter to digit or digit to letter
else if (prev_char.is_alphabetic() && curr_char.is_numeric()) ||
(prev_char.is_numeric() && curr_char.is_alphabetic()) {
camel_transitions += 1;
}
}
// If we found camelCase transitions, estimate words
if camel_transitions > 0 {
word_count = camel_transitions + 1; // +1 for the first word
}
// Strategy 2: If no camelCase detected, estimate based on character count
if word_count == 0 {
// Estimate based on typical word length (4-6 characters per word)
word_count = (alphanumeric_chars / 5).max(1);
} }
word_count word_count

102
src/tests/enhanced_ocr_tests.rs
View File

@ -38,6 +38,108 @@ mod tests {
assert_eq!(stats.sharpness, 0.8); assert_eq!(stats.sharpness, 0.8);
} }
#[test]
fn test_count_words_safely_whitespace_separated() {
let temp_dir = create_temp_dir();
let temp_path = temp_dir.path().to_str().unwrap().to_string();
let service = EnhancedOcrService::new(temp_path);
// Test normal whitespace-separated text
let text = "Hello world this is a test";
let count = service.count_words_safely(&text);
assert_eq!(count, 6);
// Test with extra whitespace
let text = " Hello world \n test ";
let count = service.count_words_safely(&text);
assert_eq!(count, 3);
}
#[test]
fn test_count_words_safely_continuous_text() {
let temp_dir = create_temp_dir();
let temp_path = temp_dir.path().to_str().unwrap().to_string();
let service = EnhancedOcrService::new(temp_path);
// Test continuous text without spaces (like some PDF extractions)
let text = "HelloWorldThisIsAContinuousText";
let count = service.count_words_safely(&text);
assert!(count > 0, "Should detect words even without whitespace");
// Test mixed alphanumeric without spaces
let text = "ABC123DEF456GHI789";
let count = service.count_words_safely(&text);
assert!(count > 0, "Should detect alphanumeric patterns as words");
}
#[test]
fn test_count_words_safely_edge_cases() {
let temp_dir = create_temp_dir();
let temp_path = temp_dir.path().to_str().unwrap().to_string();
let service = EnhancedOcrService::new(temp_path);
// Test empty text
let count = service.count_words_safely("");
assert_eq!(count, 0);
// Test only whitespace
let count = service.count_words_safely(" \n\t ");
assert_eq!(count, 0);
// Test only punctuation
let text = "!@#$%^&*()_+-=[]{}|;':\",./<>?";
let count = service.count_words_safely(&text);
// Since there are no alphabetic or alphanumeric chars, should be 0
assert_eq!(count, 0, "Pure punctuation should not count as words, got {}", count);
// Test single character
let count = service.count_words_safely("A");
assert_eq!(count, 1);
// Test mixed content with low alphanumeric ratio
let text = "A!!!B@@@C###D$$$E%%%";
let count = service.count_words_safely(&text);
assert!(count > 0, "Should detect words in mixed content");
}
#[test]
fn test_count_words_safely_large_text() {
let temp_dir = create_temp_dir();
let temp_path = temp_dir.path().to_str().unwrap().to_string();
let service = EnhancedOcrService::new(temp_path);
// Test with large text (over 1MB) to trigger sampling
let word = "test ";
let large_text = word.repeat(250_000); // Creates ~1.25MB of text
let count = service.count_words_safely(&large_text);
// Should estimate around 250,000 words (may vary due to sampling)
assert!(count > 200_000, "Should estimate large word count: got {}", count);
assert!(count <= 10_000_000, "Should cap at max limit: got {}", count);
}
#[test]
fn test_count_words_safely_fallback_patterns() {
let temp_dir = create_temp_dir();
let temp_path = temp_dir.path().to_str().unwrap().to_string();
let service = EnhancedOcrService::new(temp_path);
// Test letter transition detection
let text = "OneWordAnotherWordFinalWord";
let count = service.count_words_safely(&text);
assert!(count >= 3, "Should detect at least 3 words from transitions: got {}", count);
// Test alphanumeric estimation fallback
let text = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; // 26 chars, should estimate ~5-6 words
let count = service.count_words_safely(&text);
assert!(count >= 1 && count <= 10, "Should estimate reasonable word count: got {}", count);
// Test mixed case with numbers
let text = "ABC123def456GHI789jkl";
let count = service.count_words_safely(&text);
assert!(count >= 1, "Should detect words in mixed alphanumeric: got {}", count);
}
#[test] #[test]
fn test_ocr_result_structure() { fn test_ocr_result_structure() {
let result = OcrResult { let result = OcrResult {

293
tests/integration_pdf_word_count_tests.rs Normal file
View File

@ -0,0 +1,293 @@
#[cfg(test)]
mod pdf_word_count_integration_tests {
use readur::ocr::enhanced::EnhancedOcrService;
use readur::models::Settings;
use std::fs::File;
use std::io::Write;
use tempfile::{NamedTempFile, TempDir};
fn create_test_settings() -> Settings {
Settings::default()
}
fn create_temp_dir() -> TempDir {
TempDir::new().expect("Failed to create temp directory")
}
/// Create a mock PDF with specific text patterns for testing
fn create_mock_pdf_file(content: &str) -> NamedTempFile {
let mut temp_file = NamedTempFile::new().expect("Failed to create temp file");
// Create a minimal PDF structure that pdf-extract can read
// This is a very basic PDF that contains the specified text
let pdf_content = format!(
"%PDF-1.4\n\
1 0 obj\n\
<<\n\
/Type /Catalog\n\
/Pages 2 0 R\n\
>>\n\
endobj\n\
2 0 obj\n\
<<\n\
/Type /Pages\n\
/Kids [3 0 R]\n\
/Count 1\n\
>>\n\
endobj\n\
3 0 obj\n\
<<\n\
/Type /Page\n\
/Parent 2 0 R\n\
/Contents 4 0 R\n\
>>\n\
endobj\n\
4 0 obj\n\
<<\n\
/Length {}\n\
>>\n\
stream\n\
BT\n\
/F1 12 Tf\n\
72 720 Td\n\
({}) Tj\n\
ET\n\
endstream\n\
endobj\n\
xref\n\
0 5\n\
0000000000 65535 f \n\
0000000009 00000 n \n\
0000000074 00000 n \n\
0000000120 00000 n \n\
0000000179 00000 n \n\
trailer\n\
<<\n\
/Size 5\n\
/Root 1 0 R\n\
>>\n\
startxref\n\
{}\n\
%%EOF",
content.len() + 42, // Approximate content length
content,
300 // Approximate xref position
);
temp_file.write_all(pdf_content.as_bytes()).expect("Failed to write PDF content");
temp_file.flush().expect("Failed to flush temp file");
temp_file
}
#[tokio::test]
async fn test_pdf_extraction_with_normal_text() {
let temp_dir = create_temp_dir();
let temp_path = temp_dir.path().to_str().unwrap().to_string();
let service = EnhancedOcrService::new(temp_path);
let settings = create_test_settings();
// Create a PDF with normal spaced text
let pdf_content = "Hello world this is a test document with normal spacing";
let pdf_file = create_mock_pdf_file(pdf_content);
// Note: This test may fail because our mock PDF might not be perfectly formatted
// for pdf-extract, but it demonstrates the testing pattern
match service.extract_text_from_pdf(pdf_file.path().to_str().unwrap(), &settings).await {
Ok(result) => {
assert!(result.word_count > 0, "Should extract words from PDF with normal text");
assert!(result.confidence >= 90.0, "PDF extraction should have high confidence");
assert!(!result.text.is_empty(), "Should extract non-empty text");
}
Err(e) => {
// Mock PDF might not work with pdf-extract, but we can still test the pattern
println!("PDF extraction failed (expected with mock PDF): {}", e);
}
}
}
#[tokio::test]
async fn test_pdf_extraction_with_continuous_text() {
let temp_dir = create_temp_dir();
let temp_path = temp_dir.path().to_str().unwrap().to_string();
let service = EnhancedOcrService::new(temp_path);
let settings = create_test_settings();
// Create a PDF with continuous text (no spaces)
let pdf_content = "HelloWorldThisIsAContinuousTextWithoutSpaces";
let pdf_file = create_mock_pdf_file(pdf_content);
match service.extract_text_from_pdf(pdf_file.path().to_str().unwrap(), &settings).await {
Ok(result) => {
// The enhanced word counting should detect words even without spaces
assert!(result.word_count > 0, "Should detect words in continuous text: got {} words", result.word_count);
assert!(result.confidence >= 90.0, "PDF extraction should have high confidence");
// Verify the text was extracted
assert!(!result.text.is_empty(), "Should extract non-empty text");
assert!(result.text.contains("Hello") || result.text.contains("World"),
"Should contain expected content");
}
Err(e) => {
println!("PDF extraction failed (expected with mock PDF): {}", e);
}
}
}
#[tokio::test]
async fn test_pdf_extraction_with_mixed_content() {
let temp_dir = create_temp_dir();
let temp_path = temp_dir.path().to_str().unwrap().to_string();
let service = EnhancedOcrService::new(temp_path);
let settings = create_test_settings();
// Create a PDF with mixed content (letters, numbers, punctuation)
let pdf_content = "ABC123xyz789!@#DefGhi456";
let pdf_file = create_mock_pdf_file(pdf_content);
match service.extract_text_from_pdf(pdf_file.path().to_str().unwrap(), &settings).await {
Ok(result) => {
// Should detect alphanumeric patterns as words
assert!(result.word_count > 0, "Should detect words in mixed content: got {} words", result.word_count);
assert!(result.confidence >= 90.0, "PDF extraction should have high confidence");
}
Err(e) => {
println!("PDF extraction failed (expected with mock PDF): {}", e);
}
}
}
#[tokio::test]
async fn test_pdf_extraction_empty_content() {
let temp_dir = create_temp_dir();
let temp_path = temp_dir.path().to_str().unwrap().to_string();
let service = EnhancedOcrService::new(temp_path);
let settings = create_test_settings();
// Create a PDF with only whitespace/empty content
let pdf_content = " \n\t ";
let pdf_file = create_mock_pdf_file(pdf_content);
match service.extract_text_from_pdf(pdf_file.path().to_str().unwrap(), &settings).await {
Ok(result) => {
assert_eq!(result.word_count, 0, "Empty content should have 0 words");
assert!(result.text.trim().is_empty(), "Should extract empty/whitespace text");
}
Err(e) => {
println!("PDF extraction failed (expected with mock PDF): {}", e);
}
}
}
#[tokio::test]
async fn test_pdf_extraction_punctuation_only() {
let temp_dir = create_temp_dir();
let temp_path = temp_dir.path().to_str().unwrap().to_string();
let service = EnhancedOcrService::new(temp_path);
let settings = create_test_settings();
// Create a PDF with only punctuation
let pdf_content = "!@#$%^&*()_+-=[]{}|;':\",./<>?";
let pdf_file = create_mock_pdf_file(pdf_content);
match service.extract_text_from_pdf(pdf_file.path().to_str().unwrap(), &settings).await {
Ok(result) => {
// Pure punctuation should not count as words
assert_eq!(result.word_count, 0, "Pure punctuation should have 0 words: got {} words", result.word_count);
}
Err(e) => {
println!("PDF extraction failed (expected with mock PDF): {}", e);
}
}
}
#[tokio::test]
async fn test_pdf_quality_validation() {
let temp_dir = create_temp_dir();
let temp_path = temp_dir.path().to_str().unwrap().to_string();
let service = EnhancedOcrService::new(temp_path);
let settings = create_test_settings();
// Create a PDF with good content
let pdf_content = "This is a quality document with proper text content";
let pdf_file = create_mock_pdf_file(pdf_content);
match service.extract_text_from_pdf(pdf_file.path().to_str().unwrap(), &settings).await {
Ok(result) => {
// Test quality validation
let is_valid = service.validate_ocr_quality(&result, &settings);
if result.word_count > 0 {
assert!(is_valid, "Good quality PDF should pass validation");
} else {
assert!(!is_valid, "PDF with 0 words should fail validation");
}
// Verify OCR result structure
assert!(result.confidence >= 0.0 && result.confidence <= 100.0, "Confidence should be in valid range");
assert!(result.processing_time_ms > 0, "Should have processing time");
assert!(result.preprocessing_applied.contains(&"PDF text extraction".to_string()),
"Should indicate PDF extraction was used");
assert!(result.processed_image_path.is_none(), "PDF extraction should not produce processed image");
}
Err(e) => {
println!("PDF extraction failed (expected with mock PDF): {}", e);
}
}
}
/// Test PDF extraction with actual file-like scenarios
#[tokio::test]
async fn test_pdf_file_size_validation() {
let temp_dir = create_temp_dir();
let temp_path = temp_dir.path().to_str().unwrap().to_string();
let service = EnhancedOcrService::new(temp_path);
let settings = create_test_settings();
// Create a small PDF file to test file operations
let pdf_content = "Small test document";
let pdf_file = create_mock_pdf_file(pdf_content);
// Test that the file exists and can be read
let file_path = pdf_file.path().to_str().unwrap();
assert!(std::path::Path::new(file_path).exists(), "PDF file should exist");
// Test file size checking (this will work even if PDF extraction fails)
let metadata = tokio::fs::metadata(file_path).await.expect("Should read file metadata");
assert!(metadata.len() > 0, "PDF file should have content");
assert!(metadata.len() < 100 * 1024 * 1024, "Test PDF should be under size limit");
}
#[test]
fn test_word_counting_regression_cases() {
let temp_dir = create_temp_dir();
let temp_path = temp_dir.path().to_str().unwrap().to_string();
let service = EnhancedOcrService::new(temp_path);
// Regression test cases for the specific PDF issue
let test_cases = vec![
// Case 1: Continuous text like NDA documents
("SOCLogixNDAConfidentialityAgreement", "SOC Logix NDA type content"),
// Case 2: Mixed case and numbers
("ABC123DEF456", "Mixed alphanumeric content"),
// Case 3: Document-like text patterns
("ThisIsATestDocumentWithCamelCase", "CamelCase document text"),
// Case 4: All caps
("THISISALLCAPSTEXT", "All caps text"),
// Case 5: Mixed with punctuation
("Text.With.Dots.Between", "Text with dot separators"),
];
for (input, description) in test_cases {
let count = service.count_words_safely(input);
assert!(count > 0, "Should detect words in {}: '{}' -> {} words", description, input, count);
// Test that the counting is consistent
let count2 = service.count_words_safely(input);
assert_eq!(count, count2, "Word counting should be consistent for {}", description);
}
}
}

58
tests/test_pdfs/continuous_text.pdf Normal file
View File

@ -0,0 +1,58 @@
%PDF-1.4
1 0 obj
<<
/Type /Catalog
/Pages 2 0 R
>>
endobj
2 0 obj
<<
/Type /Pages
/Kids [3 0 R]
/Count 1
>>
endobj
3 0 obj
<<
/Type /Page
/Parent 2 0 R
/Contents 4 0 R
/MediaBox [0 0 612 792]
/Resources <<
/Font <<
/F1 <<
/Type /Font
/Subtype /Type1
/BaseFont /Helvetica
>>
>>
>>
>>
endobj
4 0 obj
<<
/Length 85
>>
stream
BT
/F1 12 Tf
72 720 Td
(HelloWorldThisIsAContinuousTextDocumentWithoutSpaces) Tj
ET
endstream
endobj
xref
0 5
0000000000 65535 f
0000000009 00000 n
0000000074 00000 n
0000000120 00000 n
0000000324 00000 n
trailer
<<
/Size 5
/Root 1 0 R
>>
startxref
458
%%EOF

68
tests/test_pdfs/edge_cases_realistic.pdf Normal file
View File

@ -0,0 +1,68 @@
%PDF-1.3
%“Œ‹ž ReportLab Generated PDF document http://www.reportlab.com
1 0 obj
<<
/F1 2 0 R
>>
endobj
2 0 obj
<<
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
>>
endobj
3 0 obj
<<
/Contents 7 0 R /MediaBox [ 0 0 612 792 ] /Parent 6 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
4 0 obj
<<
/PageMode /UseNone /Pages 6 0 R /Type /Catalog
>>
endobj
5 0 obj
<<
/Author (anonymous) /CreationDate (D:20250701003050+00'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20250701003050+00'00') /Producer (ReportLab PDF Library - www.reportlab.com)
/Subject (unspecified) /Title (untitled) /Trapped /False
>>
endobj
6 0 obj
<<
/Count 1 /Kids [ 3 0 R ] /Type /Pages
>>
endobj
7 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 435
>>
stream
Gas2FbAP0N&4Q>@`F[Y%.Ps7\=+r&.TaDN9FDI%j\;fl(SeUts3[@6I8n?$Og\5CDGQL#''qG##!t0\Z>C^)]YsU51N?FOC7&T-/BFrVn-8U.TE>KpJ67<j7lth64/J<`F1p"q#*o\-uiLfVL%_pabb7%'7`^+U%]WaC2E4LpU*X>pIGcigO]?M8F/E%`kc`e_!I@T)ejA9U6WoDt7H'Vi28;J^,8m7$:9q3l'iI&$Kd?fAaqin.kYAftnQE9*#KQ/5,X!8q,?l72@d:bYAqOBaqO@fa1UiD9W26MY^GZp@Zk#,6`]A;nO).OP(V&Grg<@)LX`fc[/f)8_8IS<O_9#b.e26?e0m*l)P"@ZLom$3T/k8Er%X!(2hc]=nib+-6=qb3$r(MrJUhItX4I/5r0k%ZO$ig1"[44WHgZ+("3o*=l>c8#~>endstream
endobj
xref
0 8
0000000000 65535 f
0000000073 00000 n
0000000104 00000 n
0000000211 00000 n
0000000404 00000 n
0000000472 00000 n
0000000768 00000 n
0000000827 00000 n
trailer
<<
/ID
[<4d1effad52f9f6c5ad297996ada120d6><4d1effad52f9f6c5ad297996ada120d6>]
% ReportLab generated PDF document -- digest (http://www.reportlab.com)
/Info 5 0 R
/Root 4 0 R
/Size 8
>>
startxref
1352
%%EOF

1
tests/test_pdfs/mixed_content.txt Normal file
View File

@ -0,0 +1 @@
Document with numbers 123 and symbols @#$ mixed with normal text.

4
tests/test_pdfs/multiline_text.txt Normal file
View File

@ -0,0 +1,4 @@
Line one with several words
Line two with more content
Line three continues the pattern
Final line ends the document

101
tests/test_pdfs/multipage_document.pdf Normal file
View File

@ -0,0 +1,101 @@
%PDF-1.4
1 0 obj
<<
/Type /Catalog
/Pages 2 0 R
>>
endobj
2 0 obj
<<
/Type /Pages
/Kids [3 0 R 4 0 R]
/Count 2
>>
endobj
3 0 obj
<<
/Type /Page
/Parent 2 0 R
/Contents 5 0 R
/MediaBox [0 0 612 792]
/Resources <<
/Font <<
/F1 <<
/Type /Font
/Subtype /Type1
/BaseFont /Helvetica
>>
>>
>>
>>
endobj
4 0 obj
<<
/Type /Page
/Parent 2 0 R
/Contents 6 0 R
/MediaBox [0 0 612 792]
/Resources <<
/Font <<
/F1 <<
/Type /Font
/Subtype /Type1
/BaseFont /Helvetica
>>
>>
>>
>>
endobj
5 0 obj
<<
/Length 200
>>
stream
BT
/F1 12 Tf
72 720 Td
(Page 1: This is the first page of a multi-page document.) Tj
0 -24 Td
(It contains multiple sentences with proper spacing.) Tj
0 -24 Td
(Each line should be counted as separate words.) Tj
0 -24 Td
(Total words on this page should be easily counted.) Tj
ET
endstream
endobj
6 0 obj
<<
/Length 180
>>
stream
BT
/F1 12 Tf
72 720 Td
(Page 2: Continuing from the previous page.) Tj
0 -24 Td
(This page also has normal text formatting.) Tj
0 -24 Td
(Word counting should work correctly here too.) Tj
0 -24 Td
(End of document with proper word boundaries.) Tj
ET
endstream
endobj
xref
0 7
0000000000 65535 f
0000000009 00000 n
0000000074 00000 n
0000000125 00000 n
0000000369 00000 n
0000000613 00000 n
0000000863 00000 n
trailer
<<
/Size 7
/Root 1 0 R
>>
startxref
1092
%%EOF

87
tests/test_pdfs/multipage_realistic.pdf Normal file
View File

@ -0,0 +1,87 @@
%PDF-1.3
%“Œ‹ž ReportLab Generated PDF document http://www.reportlab.com
1 0 obj
<<
/F1 2 0 R
>>
endobj
2 0 obj
<<
/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
>>
endobj
3 0 obj
<<
/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
4 0 obj
<<
/Contents 9 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
>> /Rotate 0 /Trans <<
>>
/Type /Page
>>
endobj
5 0 obj
<<
/PageMode /UseNone /Pages 7 0 R /Type /Catalog
>>
endobj
6 0 obj
<<
/Author (anonymous) /CreationDate (D:20250701003050+00'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20250701003050+00'00') /Producer (ReportLab PDF Library - www.reportlab.com)
/Subject (unspecified) /Title (untitled) /Trapped /False
>>
endobj
7 0 obj
<<
/Count 2 /Kids [ 3 0 R 4 0 R ] /Type /Pages
>>
endobj
8 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 406
>>
stream
Gas30:J\kN(^BL,5/hSB1T0TCo4XYNM4,GU4MNCg`DL<!B(=XQG1=`gCYCUZ.6ejp"Rc'uVe8j/:D.k)!b)L>6Hgfua>[qrB]-MdM:E<`236A!g<s:p4Q>$1D67*\dA.-<X\G[t)VoAFLAZY9q$1&56rkXdmo4"c-H(S7@snYMh,1YZGL`lO\I?b=pmP$(QcQ\(JM'UVWS/(Jk)<%(N=LaR'uoVG9TdR/'c!fi$rt$L$9QLjZtq3gAA+[%8`T#eMO1kB?ed%/L)nTA'F\WK^mrphlo1.]Go`/kFoh7IfU)B\eiOlr7m-9t9P7kZ(X"PS.BFTA^S/b=T48CfI>ck2're$?]L,EP&5d/rS9LEhgfJ2<>Ml"DM/7&6N2>sT:If`6;H#EbotJhg,Xt"A)9AH~>endstream
endobj
9 0 obj
<<
/Filter [ /ASCII85Decode /FlateDecode ] /Length 402
>>
stream
Gas2D95h[$&;9NMME/*[U3.TSE%?`.UfKfsUgA@%p*OcNCh,UirdAXH6Os?'nlkA0[kkqk&.UeZU&c#im[kA!K"M)QN$cX'Z-!<Fchc$?_/pIl)r.N?8P%uG)XWf-PqGp9dpR$,Y>"6n#B#\(+M[f/P'3)&;@^<pijCS@\:Z]JiAE_<4c9%.JR=EiUW+>>b?,'Zg*ikPX%Z_j*/r^.VR7[sI6pAgi/P96@iar(*@^!T-gg)A=tEf3L88:A:@[oOQ9EO=E/-bo]$*gRNq^(/Tn4jnEQa%`lVOW661El\S?a:p%lN+j&6ql\fQ:sDb@Pi=@diu[i.aRS>1Q";cC,d,i-KHrY6Z;u\69bu7;d$n'f;9ZdYP=<!T9VueH;R`M+n7ZEi[:[KjjHY\5TBt~>endstream
endobj
xref
0 10
0000000000 65535 f
0000000073 00000 n
0000000104 00000 n
0000000211 00000 n
0000000404 00000 n
0000000597 00000 n
0000000665 00000 n
0000000961 00000 n
0000001026 00000 n
0000001522 00000 n
trailer
<<
/ID
[<098a3737c0449fe862826c7afae59697><098a3737c0449fe862826c7afae59697>]
% ReportLab generated PDF document -- digest (http://www.reportlab.com)
/Info 6 0 R
/Root 5 0 R
/Size 10
>>
startxref
2014
%%EOF

1
tests/test_pdfs/normal_spacing.txt Normal file
View File

@ -0,0 +1 @@
This is a normal document with proper word spacing and punctuation.

58
tests/test_pdfs/normal_text.pdf Normal file
View File

@ -0,0 +1,58 @@
%PDF-1.4
1 0 obj
<<
/Type /Catalog
/Pages 2 0 R
>>
endobj
2 0 obj
<<
/Type /Pages
/Kids [3 0 R]
/Count 1
>>
endobj
3 0 obj
<<
/Type /Page
/Parent 2 0 R
/Contents 4 0 R
/MediaBox [0 0 612 792]
/Resources <<
/Font <<
/F1 <<
/Type /Font
/Subtype /Type1
/BaseFont /Helvetica
>>
>>
>>
>>
endobj
4 0 obj
<<
/Length 75
>>
stream
BT
/F1 12 Tf
72 720 Td
(This is a normal document with proper word spacing) Tj
ET
endstream
endobj
xref
0 5
0000000000 65535 f
0000000009 00000 n
0000000074 00000 n
0000000120 00000 n
0000000324 00000 n
trailer
<<
/Size 5
/Root 1 0 R
>>
startxref
448
%%EOF

64
tests/test_pdfs/problematic_encoding.pdf Normal file
View File

@ -0,0 +1,64 @@
%PDF-1.4
1 0 obj
<<
/Type /Catalog
/Pages 2 0 R
>>
endobj
2 0 obj
<<
/Type /Pages
/Kids [3 0 R]
/Count 1
>>
endobj
3 0 obj
<<
/Type /Page
/Parent 2 0 R
/Contents 4 0 R
/MediaBox [0 0 612 792]
/Resources <<
/Font <<
/F1 <<
/Type /Font
/Subtype /Type1
/BaseFont /Helvetica
>>
>>
>>
>>
endobj
4 0 obj
<<
/Length 165
>>
stream
BT
/F1 12 Tf
72 720 Td
(Text with special characters: caf\351 na\357ve r\351sum\351) Tj
0 -24 Td
(Unicode characters: \u2022 \u2013 \u2014 \u201C \u201D) Tj
0 -24 Td
(Mixed content: ABC123 def456 GHI789) Tj
0 -24 Td
(Normal text: This should work fine.) Tj
ET
endstream
endobj
xref
0 5
0000000000 65535 f
0000000009 00000 n
0000000074 00000 n
0000000120 00000 n
0000000324 00000 n
trailer
<<
/Size 5
/Root 1 0 R
>>
startxref
538
%%EOF

1
tests/test_pdfs/special_chars.txt Normal file
View File

@ -0,0 +1 @@
Text with special characters: café naïve résumé — and 'quotes' • bullets