1607 lines
65 KiB
Rust
1607 lines
65 KiB
Rust
/*!
|
||
* File Processing Pipeline Integration Tests
|
||
*
|
||
* Tests the complete file processing pipeline including:
|
||
* - File upload and validation
|
||
* - Thumbnail generation
|
||
* - Image preprocessing
|
||
* - OCR processing stages
|
||
* - Text extraction and indexing
|
||
* - File format support
|
||
* - Error recovery in processing
|
||
* - Pipeline performance monitoring
|
||
* - Resource cleanup
|
||
*/
|
||
|
||
use reqwest::Client;
|
||
use serde_json::Value;
|
||
use std::time::{Duration, Instant};
|
||
use tokio::time::sleep;
|
||
use uuid::Uuid;
|
||
|
||
use readur::models::{CreateUser, LoginRequest, LoginResponse, UserRole, DocumentResponse};
|
||
use readur::routes::documents::types::PaginatedDocumentsResponse;
|
||
use readur::routes::documents::types::DocumentUploadResponse;
|
||
|
||
fn get_base_url() -> String {
|
||
std::env::var("API_URL").unwrap_or_else(|_| "http://localhost:8000".to_string())
|
||
}
|
||
const PROCESSING_TIMEOUT: Duration = Duration::from_secs(120);
|
||
|
||
/// Test image structure for pipeline tests
|
||
struct TestImage {
|
||
filename: String,
|
||
path: String,
|
||
mime_type: String,
|
||
expected_content: Option<String>,
|
||
}
|
||
|
||
impl TestImage {
|
||
fn load_data(&self) -> Result<Vec<u8>, std::io::Error> {
|
||
// Return empty data for test - this would normally read a file
|
||
Ok(vec![])
|
||
}
|
||
}
|
||
|
||
/// Test client for file processing pipeline tests
|
||
struct FileProcessingTestClient {
|
||
client: Client,
|
||
token: Option<String>,
|
||
user_id: Option<String>,
|
||
}
|
||
|
||
impl FileProcessingTestClient {
|
||
fn new() -> Self {
|
||
Self {
|
||
client: Client::new(),
|
||
token: None,
|
||
user_id: None,
|
||
}
|
||
}
|
||
|
||
/// Setup test user
|
||
async fn setup_user(&mut self) -> Result<String, Box<dyn std::error::Error>> {
|
||
let timestamp = std::time::SystemTime::now()
|
||
.duration_since(std::time::UNIX_EPOCH)
|
||
.unwrap()
|
||
.as_nanos();
|
||
let random_suffix = uuid::Uuid::new_v4().to_string().replace("-", "")[..8].to_string();
|
||
let username = format!("file_proc_test_{}_{}", timestamp, random_suffix);
|
||
let email = format!("file_proc_test_{}@example.com", timestamp);
|
||
let password = "fileprocessingpassword123";
|
||
|
||
// Register user with retry logic
|
||
let user_data = CreateUser {
|
||
username: username.clone(),
|
||
email: email.clone(),
|
||
password: password.to_string(),
|
||
role: Some(UserRole::User),
|
||
};
|
||
|
||
let mut retry_count = 0;
|
||
let register_response = loop {
|
||
match self.client
|
||
.post(&format!("{}/api/auth/register", get_base_url()))
|
||
.json(&user_data)
|
||
.timeout(Duration::from_secs(10))
|
||
.send()
|
||
.await
|
||
{
|
||
Ok(resp) => break resp,
|
||
Err(e) => {
|
||
retry_count += 1;
|
||
if retry_count >= 3 {
|
||
return Err(format!("Registration failed after 3 retries: {}", e).into());
|
||
}
|
||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||
}
|
||
}
|
||
};
|
||
|
||
if !register_response.status().is_success() {
|
||
let status = register_response.status();
|
||
let text = register_response.text().await.unwrap_or_else(|_| "No response body".to_string());
|
||
return Err(format!("Registration failed with status {}: {}", status, text).into());
|
||
}
|
||
|
||
// Login to get token
|
||
let login_data = LoginRequest {
|
||
username: username.clone(),
|
||
password: password.to_string(),
|
||
};
|
||
|
||
let login_response = self.client
|
||
.post(&format!("{}/api/auth/login", get_base_url()))
|
||
.json(&login_data)
|
||
.send()
|
||
.await?;
|
||
|
||
if !login_response.status().is_success() {
|
||
return Err(format!("Login failed: {}", login_response.text().await?).into());
|
||
}
|
||
|
||
let login_result: LoginResponse = login_response.json().await?;
|
||
self.token = Some(login_result.token.clone());
|
||
|
||
// Get user info
|
||
let me_response = self.client
|
||
.get(&format!("{}/api/auth/me", get_base_url()))
|
||
.header("Authorization", format!("Bearer {}", login_result.token))
|
||
.send()
|
||
.await?;
|
||
|
||
if me_response.status().is_success() {
|
||
let user_info: Value = me_response.json().await?;
|
||
self.user_id = user_info["id"].as_str().map(|s| s.to_string());
|
||
}
|
||
|
||
Ok(login_result.token)
|
||
}
|
||
|
||
/// Upload a file with specific content and MIME type
|
||
async fn upload_file(&self, content: &str, filename: &str, mime_type: &str) -> Result<DocumentUploadResponse, Box<dyn std::error::Error>> {
|
||
println!("🔍 DEBUG: Uploading file: {} with MIME type: {}", filename, mime_type);
|
||
let token = self.token.as_ref().ok_or("Not authenticated")?;
|
||
|
||
let part = reqwest::multipart::Part::text(content.to_string())
|
||
.file_name(filename.to_string())
|
||
.mime_str(mime_type)?;
|
||
let form = reqwest::multipart::Form::new()
|
||
.part("file", part);
|
||
|
||
let response = self.client
|
||
.post(&format!("{}/api/documents", get_base_url()))
|
||
.header("Authorization", format!("Bearer {}", token))
|
||
.multipart(form)
|
||
.send()
|
||
.await?;
|
||
|
||
let status = response.status();
|
||
if !status.is_success() {
|
||
let error_text = response.text().await?;
|
||
println!("🔴 DEBUG: Upload failed with status {}: {}", status, error_text);
|
||
return Err(format!("Upload failed: {}", error_text).into());
|
||
}
|
||
|
||
let response_text = response.text().await?;
|
||
println!("🟢 DEBUG: Upload response: {}", response_text);
|
||
|
||
let document: DocumentUploadResponse = serde_json::from_str(&response_text)?;
|
||
println!("✅ DEBUG: Successfully parsed document: {}", document.id);
|
||
Ok(document)
|
||
}
|
||
|
||
/// Upload binary file content
|
||
async fn upload_binary_file(&self, content: Vec<u8>, filename: &str, mime_type: &str) -> Result<DocumentUploadResponse, Box<dyn std::error::Error>> {
|
||
let token = self.token.as_ref().ok_or("Not authenticated")?;
|
||
|
||
let part = reqwest::multipart::Part::bytes(content)
|
||
.file_name(filename.to_string())
|
||
.mime_str(mime_type)?;
|
||
let form = reqwest::multipart::Form::new()
|
||
.part("file", part);
|
||
|
||
let response = self.client
|
||
.post(&format!("{}/api/documents", get_base_url()))
|
||
.header("Authorization", format!("Bearer {}", token))
|
||
.multipart(form)
|
||
.send()
|
||
.await?;
|
||
|
||
let status = response.status();
|
||
if !status.is_success() {
|
||
let error_text = response.text().await?;
|
||
println!("🔴 DEBUG: Binary upload failed with status {}: {}", status, error_text);
|
||
return Err(format!("Binary upload failed: {}", error_text).into());
|
||
}
|
||
|
||
let response_text = response.text().await?;
|
||
println!("🟢 DEBUG: Binary upload response: {}", response_text);
|
||
|
||
let document: DocumentUploadResponse = serde_json::from_str(&response_text)?;
|
||
println!("✅ DEBUG: Successfully parsed binary document: {}", document.id);
|
||
Ok(document)
|
||
}
|
||
|
||
/// Upload a file with multiple OCR languages specified
|
||
async fn upload_file_with_languages(&self, content: &str, filename: &str, mime_type: &str, languages: &[&str]) -> Result<DocumentUploadResponse, Box<dyn std::error::Error>> {
|
||
println!("🔍 DEBUG: Uploading file: {} with MIME type: {} and languages: {:?}", filename, mime_type, languages);
|
||
let token = self.token.as_ref().ok_or("Not authenticated")?;
|
||
|
||
let part = reqwest::multipart::Part::text(content.to_string())
|
||
.file_name(filename.to_string())
|
||
.mime_str(mime_type)?;
|
||
|
||
let mut form = reqwest::multipart::Form::new()
|
||
.part("file", part);
|
||
|
||
// Add multiple language parameters
|
||
for (index, language) in languages.iter().enumerate() {
|
||
form = form.text(format!("ocr_languages[{}]", index), language.to_string());
|
||
}
|
||
|
||
let response = self.client
|
||
.post(&format!("{}/api/documents", get_base_url()))
|
||
.header("Authorization", format!("Bearer {}", token))
|
||
.multipart(form)
|
||
.send()
|
||
.await?;
|
||
|
||
let status = response.status();
|
||
if !status.is_success() {
|
||
let error_text = response.text().await?;
|
||
println!("🔴 DEBUG: Multi-language upload failed with status {}: {}", status, error_text);
|
||
return Err(format!("Multi-language upload failed: {}", error_text).into());
|
||
}
|
||
|
||
let response_text = response.text().await?;
|
||
println!("🟢 DEBUG: Multi-language upload response: {}", response_text);
|
||
|
||
let document: DocumentUploadResponse = serde_json::from_str(&response_text)?;
|
||
println!("✅ DEBUG: Successfully parsed multi-language document: {}", document.id);
|
||
Ok(document)
|
||
}
|
||
|
||
/// Upload binary file content with multiple OCR languages
|
||
async fn upload_binary_file_with_languages(&self, content: Vec<u8>, filename: &str, mime_type: &str, languages: &[&str]) -> Result<DocumentUploadResponse, Box<dyn std::error::Error>> {
|
||
println!("🔍 DEBUG: Uploading binary file: {} with MIME type: {} and languages: {:?}", filename, mime_type, languages);
|
||
let token = self.token.as_ref().ok_or("Not authenticated")?;
|
||
|
||
let part = reqwest::multipart::Part::bytes(content)
|
||
.file_name(filename.to_string())
|
||
.mime_str(mime_type)?;
|
||
|
||
let mut form = reqwest::multipart::Form::new()
|
||
.part("file", part);
|
||
|
||
// Add multiple language parameters
|
||
for (index, language) in languages.iter().enumerate() {
|
||
form = form.text(format!("ocr_languages[{}]", index), language.to_string());
|
||
}
|
||
|
||
let response = self.client
|
||
.post(&format!("{}/api/documents", get_base_url()))
|
||
.header("Authorization", format!("Bearer {}", token))
|
||
.multipart(form)
|
||
.send()
|
||
.await?;
|
||
|
||
let status = response.status();
|
||
if !status.is_success() {
|
||
let error_text = response.text().await?;
|
||
println!("🔴 DEBUG: Multi-language binary upload failed with status {}: {}", status, error_text);
|
||
return Err(format!("Multi-language binary upload failed: {}", error_text).into());
|
||
}
|
||
|
||
let response_text = response.text().await?;
|
||
println!("🟢 DEBUG: Multi-language binary upload response: {}", response_text);
|
||
|
||
let document: DocumentUploadResponse = serde_json::from_str(&response_text)?;
|
||
println!("✅ DEBUG: Successfully parsed multi-language binary document: {}", document.id);
|
||
Ok(document)
|
||
}
|
||
|
||
/// Wait for document processing to complete
|
||
async fn wait_for_processing(&self, document_id: &str) -> Result<DocumentResponse, Box<dyn std::error::Error>> {
|
||
println!("🔍 DEBUG: Waiting for processing of document: {}", document_id);
|
||
let token = self.token.as_ref().ok_or("Not authenticated")?;
|
||
let start = Instant::now();
|
||
|
||
while start.elapsed() < PROCESSING_TIMEOUT {
|
||
let response = self.client
|
||
.get(&format!("{}/api/documents", get_base_url()))
|
||
.header("Authorization", format!("Bearer {}", token))
|
||
.send()
|
||
.await?;
|
||
|
||
if response.status().is_success() {
|
||
let paginated_response: PaginatedDocumentsResponse = response.json().await?;
|
||
let documents = paginated_response.documents;
|
||
|
||
if let Some(doc) = documents.iter().find(|d| d.id.to_string() == document_id) {
|
||
println!("📄 DEBUG: Found document with OCR status: {:?}", doc.ocr_status);
|
||
match doc.ocr_status.as_deref() {
|
||
Some("completed") => {
|
||
// Create a copy of the document since we can't clone it
|
||
let doc_copy = DocumentResponse {
|
||
id: doc.id,
|
||
filename: doc.filename.clone(),
|
||
original_filename: doc.original_filename.clone(),
|
||
file_path: doc.file_path.clone(),
|
||
file_size: doc.file_size,
|
||
mime_type: doc.mime_type.clone(),
|
||
tags: doc.tags.clone(),
|
||
labels: doc.labels.clone(),
|
||
created_at: doc.created_at,
|
||
updated_at: doc.updated_at,
|
||
user_id: doc.user_id,
|
||
username: doc.username.clone(),
|
||
file_hash: doc.file_hash.clone(),
|
||
has_ocr_text: doc.has_ocr_text,
|
||
ocr_confidence: doc.ocr_confidence,
|
||
ocr_word_count: doc.ocr_word_count,
|
||
ocr_processing_time_ms: doc.ocr_processing_time_ms,
|
||
ocr_status: doc.ocr_status.clone(),
|
||
original_created_at: doc.original_created_at,
|
||
original_modified_at: doc.original_modified_at,
|
||
source_path: doc.source_path.clone(),
|
||
source_type: doc.source_type.clone(),
|
||
source_id: doc.source_id,
|
||
file_permissions: doc.file_permissions,
|
||
file_owner: doc.file_owner.clone(),
|
||
file_group: doc.file_group.clone(),
|
||
source_metadata: doc.source_metadata.clone(),
|
||
};
|
||
return Ok(doc_copy);
|
||
}
|
||
Some("failed") => return Err("Processing failed".into()),
|
||
_ => {
|
||
sleep(Duration::from_millis(500)).await;
|
||
continue;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
sleep(Duration::from_millis(500)).await;
|
||
}
|
||
|
||
Err("Processing timeout".into())
|
||
}
|
||
|
||
/// Get document thumbnail
|
||
async fn get_thumbnail(&self, document_id: &str) -> Result<(reqwest::StatusCode, Vec<u8>), Box<dyn std::error::Error>> {
|
||
let token = self.token.as_ref().ok_or("Not authenticated")?;
|
||
|
||
let response = self.client
|
||
.get(&format!("{}/api/documents/{}/thumbnail", get_base_url(), document_id))
|
||
.header("Authorization", format!("Bearer {}", token))
|
||
.send()
|
||
.await?;
|
||
|
||
let status = response.status();
|
||
let bytes = response.bytes().await?.to_vec();
|
||
|
||
Ok((status, bytes))
|
||
}
|
||
|
||
/// Get processed image
|
||
async fn get_processed_image(&self, document_id: &str) -> Result<(reqwest::StatusCode, Vec<u8>), Box<dyn std::error::Error>> {
|
||
let token = self.token.as_ref().ok_or("Not authenticated")?;
|
||
|
||
let response = self.client
|
||
.get(&format!("{}/api/documents/{}/processed-image", get_base_url(), document_id))
|
||
.header("Authorization", format!("Bearer {}", token))
|
||
.send()
|
||
.await?;
|
||
|
||
let status = response.status();
|
||
let bytes = response.bytes().await?.to_vec();
|
||
|
||
Ok((status, bytes))
|
||
}
|
||
|
||
/// Get OCR results
|
||
async fn get_ocr_results(&self, document_id: &str) -> Result<Value, Box<dyn std::error::Error>> {
|
||
let token = self.token.as_ref().ok_or("Not authenticated")?;
|
||
|
||
let response = self.client
|
||
.get(&format!("{}/api/documents/{}/ocr", get_base_url(), document_id))
|
||
.header("Authorization", format!("Bearer {}", token))
|
||
.send()
|
||
.await?;
|
||
|
||
if !response.status().is_success() {
|
||
return Err(format!("OCR retrieval failed: {}", response.text().await?).into());
|
||
}
|
||
|
||
let ocr_data: Value = response.json().await?;
|
||
Ok(ocr_data)
|
||
}
|
||
|
||
/// Get all documents for the authenticated user
|
||
async fn get_documents(&self) -> Result<Vec<DocumentResponse>, Box<dyn std::error::Error>> {
|
||
let token = self.token.as_ref().ok_or("Not authenticated")?;
|
||
|
||
let response = self.client
|
||
.get(&format!("{}/api/documents", get_base_url()))
|
||
.header("Authorization", format!("Bearer {}", token))
|
||
.send()
|
||
.await?;
|
||
|
||
if !response.status().is_success() {
|
||
return Err(format!("Get documents failed: {}", response.text().await?).into());
|
||
}
|
||
|
||
let paginated_response: PaginatedDocumentsResponse = response.json().await?;
|
||
let documents = paginated_response.documents;
|
||
Ok(documents)
|
||
}
|
||
|
||
/// Download original file
|
||
async fn download_file(&self, document_id: &str) -> Result<(reqwest::StatusCode, Vec<u8>), Box<dyn std::error::Error>> {
|
||
let token = self.token.as_ref().ok_or("Not authenticated")?;
|
||
|
||
let response = self.client
|
||
.get(&format!("{}/api/documents/{}/download", get_base_url(), document_id))
|
||
.header("Authorization", format!("Bearer {}", token))
|
||
.send()
|
||
.await?;
|
||
|
||
let status = response.status();
|
||
let bytes = response.bytes().await?.to_vec();
|
||
|
||
Ok((status, bytes))
|
||
}
|
||
|
||
/// View file in browser
|
||
async fn view_file(&self, document_id: &str) -> Result<reqwest::StatusCode, Box<dyn std::error::Error>> {
|
||
let token = self.token.as_ref().ok_or("Not authenticated")?;
|
||
|
||
let response = self.client
|
||
.get(&format!("{}/api/documents/{}/view", get_base_url(), document_id))
|
||
.header("Authorization", format!("Bearer {}", token))
|
||
.send()
|
||
.await?;
|
||
|
||
Ok(response.status())
|
||
}
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_text_file_processing_pipeline() {
|
||
println!("📄 Testing text file processing pipeline...");
|
||
|
||
let mut client = FileProcessingTestClient::new();
|
||
client.setup_user().await
|
||
.expect("Failed to setup test user");
|
||
|
||
println!("✅ User setup complete");
|
||
|
||
// Upload a text file
|
||
let text_content = r#"This is a test document for the file processing pipeline.
|
||
It contains multiple lines of text that should be processed correctly.
|
||
|
||
Key features to test:
|
||
1. Text extraction
|
||
2. OCR processing (even for text files)
|
||
3. Thumbnail generation
|
||
4. File storage and retrieval
|
||
|
||
The document should be indexed and searchable.
|
||
Processing time should be tracked.
|
||
All pipeline stages should complete successfully.
|
||
|
||
End of test document."#;
|
||
|
||
let document = client.upload_file(text_content, "test_pipeline.txt", "text/plain").await
|
||
.expect("Failed to upload text file");
|
||
|
||
let document_id = document.id.to_string();
|
||
println!("✅ Text file uploaded: {}", document_id);
|
||
|
||
// Validate initial document properties
|
||
assert_eq!(document.mime_type, "text/plain");
|
||
assert!(document.file_size > 0);
|
||
assert_eq!(document.filename, "test_pipeline.txt");
|
||
|
||
// Wait for processing to complete
|
||
let processed_doc = client.wait_for_processing(&document_id).await
|
||
.expect("Failed to wait for processing");
|
||
|
||
assert_eq!(processed_doc.ocr_status.as_deref(), Some("completed"));
|
||
println!("✅ Text file processing completed");
|
||
|
||
// Test file download
|
||
let (download_status, downloaded_content) = client.download_file(&document_id).await
|
||
.expect("Failed to download file");
|
||
|
||
assert!(download_status.is_success());
|
||
assert!(!downloaded_content.is_empty());
|
||
let downloaded_text = String::from_utf8_lossy(&downloaded_content);
|
||
assert!(downloaded_text.contains("test document for the file processing pipeline"));
|
||
println!("✅ File download successful");
|
||
|
||
// Test file view
|
||
let view_status = client.view_file(&document_id).await
|
||
.expect("Failed to view file");
|
||
|
||
println!("✅ File view status: {}", view_status);
|
||
|
||
// Test OCR results
|
||
let ocr_results = client.get_ocr_results(&document_id).await
|
||
.expect("Failed to get OCR results");
|
||
|
||
assert_eq!(ocr_results["id"], document_id);
|
||
assert_eq!(ocr_results["has_ocr_text"], true);
|
||
|
||
if let Some(ocr_text) = ocr_results["ocr_text"].as_str() {
|
||
assert!(!ocr_text.is_empty());
|
||
assert!(ocr_text.contains("test document"));
|
||
println!("✅ OCR text extracted: {} characters", ocr_text.len());
|
||
}
|
||
|
||
// Validate OCR metadata
|
||
if ocr_results["ocr_confidence"].is_number() {
|
||
let confidence = ocr_results["ocr_confidence"].as_f64().unwrap();
|
||
assert!((0.0..=100.0).contains(&confidence));
|
||
println!("✅ OCR confidence: {:.1}%", confidence);
|
||
}
|
||
|
||
if ocr_results["ocr_word_count"].is_number() {
|
||
let word_count = ocr_results["ocr_word_count"].as_i64().unwrap();
|
||
assert!(word_count > 0);
|
||
println!("✅ OCR word count: {}", word_count);
|
||
}
|
||
|
||
if ocr_results["ocr_processing_time_ms"].is_number() {
|
||
let processing_time = ocr_results["ocr_processing_time_ms"].as_i64().unwrap();
|
||
assert!(processing_time >= 0);
|
||
println!("✅ OCR processing time: {}ms", processing_time);
|
||
}
|
||
|
||
// Test thumbnail generation
|
||
let (thumbnail_status, thumbnail_data) = client.get_thumbnail(&document_id).await
|
||
.expect("Failed to get thumbnail");
|
||
|
||
if thumbnail_status.is_success() {
|
||
assert!(!thumbnail_data.is_empty());
|
||
println!("✅ Thumbnail generated: {} bytes", thumbnail_data.len());
|
||
} else {
|
||
println!("ℹ️ Thumbnail not available for text file: {}", thumbnail_status);
|
||
}
|
||
|
||
println!("🎉 Text file processing pipeline test passed!");
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_multiple_file_format_support() {
|
||
println!("📁 Testing multiple file format support...");
|
||
|
||
let mut client = FileProcessingTestClient::new();
|
||
client.setup_user().await
|
||
.expect("Failed to setup test user");
|
||
|
||
println!("✅ User setup complete");
|
||
|
||
// Test different file formats
|
||
let test_files = vec![
|
||
("text/plain", "test.txt", "Plain text file for format testing."),
|
||
("text/csv", "test.csv", "name,age,city\nJohn,30,NYC\nJane,25,LA"),
|
||
("application/json", "test.json", r#"{"test": "data", "format": "json"}"#),
|
||
("text/xml", "test.xml", "<?xml version=\"1.0\"?><root><test>data</test></root>"),
|
||
("text/markdown", "test.md", "# Test Markdown\n\nThis is **bold** text."),
|
||
];
|
||
|
||
let mut uploaded_documents = Vec::new();
|
||
|
||
// Upload all test files
|
||
for (mime_type, filename, content) in &test_files {
|
||
println!("📤 Uploading {} file...", mime_type);
|
||
|
||
match client.upload_file(content, filename, mime_type).await {
|
||
Ok(document) => {
|
||
println!("✅ Uploaded {}: {}", filename, document.id);
|
||
uploaded_documents.push((document, mime_type, filename, content));
|
||
}
|
||
Err(e) => {
|
||
println!("⚠️ Failed to upload {}: {}", filename, e);
|
||
}
|
||
}
|
||
}
|
||
|
||
assert!(!uploaded_documents.is_empty(), "At least some files should upload successfully");
|
||
println!("✅ Uploaded {} files", uploaded_documents.len());
|
||
|
||
// Test processing for each uploaded file
|
||
for (document, mime_type, filename, original_content) in &uploaded_documents {
|
||
println!("🔄 Processing {} ({})...", filename, mime_type);
|
||
|
||
let document_id = document.id.to_string();
|
||
|
||
// Wait for processing (with shorter timeout for multiple files)
|
||
match client.wait_for_processing(&document_id).await {
|
||
Ok(processed_doc) => {
|
||
println!("✅ {} processed successfully", filename);
|
||
|
||
// Test OCR results
|
||
if let Ok(ocr_results) = client.get_ocr_results(&document_id).await {
|
||
assert_eq!(ocr_results["id"], document_id);
|
||
|
||
if ocr_results["has_ocr_text"] == true {
|
||
if let Some(ocr_text) = ocr_results["ocr_text"].as_str() {
|
||
assert!(!ocr_text.is_empty());
|
||
|
||
// Verify OCR text contains some original content
|
||
let content_words: Vec<&str> = original_content.split_whitespace().collect();
|
||
if !content_words.is_empty() {
|
||
let first_word = content_words[0];
|
||
if first_word.len() > 2 { // Only check meaningful words
|
||
println!("✅ {} OCR text contains expected content", filename);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Test file download
|
||
if let Ok((download_status, _)) = client.download_file(&document_id).await {
|
||
if download_status.is_success() {
|
||
println!("✅ {} download successful", filename);
|
||
}
|
||
}
|
||
}
|
||
Err(e) => {
|
||
println!("⚠️ {} processing failed: {}", filename, e);
|
||
}
|
||
}
|
||
}
|
||
|
||
println!("🎉 Multiple file format support test completed!");
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_image_processing_pipeline() {
|
||
println!("🖼️ Testing image processing pipeline...");
|
||
|
||
let mut client = FileProcessingTestClient::new();
|
||
client.setup_user().await
|
||
.expect("Failed to setup test user");
|
||
|
||
println!("✅ User setup complete");
|
||
|
||
// Create a simple test image (minimal PNG)
|
||
// This is a 1x1 pixel transparent PNG
|
||
let png_data = vec![
|
||
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D,
|
||
0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01,
|
||
0x08, 0x06, 0x00, 0x00, 0x00, 0x1F, 0x15, 0xC4, 0x89, 0x00, 0x00, 0x00,
|
||
0x0B, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9C, 0x63, 0x00, 0x01, 0x00, 0x00,
|
||
0x05, 0x00, 0x01, 0x0D, 0x0A, 0x2D, 0xB4, 0x00, 0x00, 0x00, 0x00, 0x49,
|
||
0x45, 0x4E, 0x44, 0xAE, 0x42, 0x60, 0x82
|
||
];
|
||
|
||
let document = client.upload_binary_file(png_data.clone(), "test_image.png", "image/png").await
|
||
.expect("Failed to upload PNG image");
|
||
|
||
let document_id = document.id.to_string();
|
||
println!("✅ PNG image uploaded: {}", document_id);
|
||
|
||
// Validate image document properties
|
||
assert_eq!(document.mime_type, "image/png");
|
||
assert!(document.file_size > 0);
|
||
assert_eq!(document.filename, "test_image.png");
|
||
|
||
// Wait for processing - note that minimal images might fail OCR
|
||
let processed_result = client.wait_for_processing(&document_id).await;
|
||
|
||
let processed_doc = match processed_result {
|
||
Ok(doc) => doc,
|
||
Err(e) => {
|
||
// For minimal test images, OCR might fail which is acceptable
|
||
println!("⚠️ Image processing failed (expected for minimal test images): {}", e);
|
||
|
||
// Get the document status directly
|
||
let response = client.client
|
||
.get(&format!("{}/api/documents", get_base_url()))
|
||
.header("Authorization", format!("Bearer {}", client.token.as_ref().unwrap()))
|
||
.send()
|
||
.await
|
||
.expect("Failed to get documents");
|
||
|
||
let paginated_response: PaginatedDocumentsResponse = response.json().await
|
||
.expect("Failed to parse response");
|
||
let documents = paginated_response.documents;
|
||
|
||
documents.into_iter()
|
||
.find(|d| d.id.to_string() == document_id)
|
||
.expect("Document not found")
|
||
}
|
||
};
|
||
|
||
println!("✅ Image processing completed with status: {:?}", processed_doc.ocr_status);
|
||
|
||
// Test thumbnail generation
|
||
let (thumbnail_status, thumbnail_data) = client.get_thumbnail(&document_id).await
|
||
.expect("Failed to get thumbnail");
|
||
|
||
if thumbnail_status.is_success() {
|
||
assert!(!thumbnail_data.is_empty());
|
||
println!("✅ Image thumbnail generated: {} bytes", thumbnail_data.len());
|
||
|
||
// Validate thumbnail is different from original (usually smaller or different format)
|
||
if thumbnail_data != png_data {
|
||
println!("✅ Thumbnail is processed (different from original)");
|
||
}
|
||
} else {
|
||
println!("ℹ️ Thumbnail generation failed: {}", thumbnail_status);
|
||
}
|
||
|
||
// Test processed image
|
||
let (processed_status, processed_data) = client.get_processed_image(&document_id).await
|
||
.expect("Failed to get processed image");
|
||
|
||
if processed_status.is_success() {
|
||
assert!(!processed_data.is_empty());
|
||
println!("✅ Processed image available: {} bytes", processed_data.len());
|
||
} else {
|
||
println!("ℹ️ Processed image not available: {}", processed_status);
|
||
}
|
||
|
||
// Test OCR on image
|
||
let ocr_results = client.get_ocr_results(&document_id).await
|
||
.expect("Failed to get OCR results for image");
|
||
|
||
assert_eq!(ocr_results["id"], document_id);
|
||
|
||
// Image might not have text, so OCR could be empty
|
||
if ocr_results["has_ocr_text"] == true {
|
||
println!("✅ Image OCR completed with text");
|
||
} else {
|
||
println!("ℹ️ Image OCR completed but no text found (expected for test image)");
|
||
}
|
||
|
||
// Test image download
|
||
let (download_status, downloaded_data) = client.download_file(&document_id).await
|
||
.expect("Failed to download image");
|
||
|
||
assert!(download_status.is_success());
|
||
assert_eq!(downloaded_data, png_data);
|
||
println!("✅ Image download matches original");
|
||
|
||
println!("🎉 Image processing pipeline test passed!");
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_processing_error_recovery() {
|
||
println!("🔧 Testing processing error recovery...");
|
||
|
||
let mut client = FileProcessingTestClient::new();
|
||
client.setup_user().await
|
||
.expect("Failed to setup test user");
|
||
|
||
println!("✅ User setup complete");
|
||
|
||
// Test 1: Empty file
|
||
println!("🔍 Testing empty file processing...");
|
||
|
||
let empty_result = client.upload_file("", "empty.txt", "text/plain").await;
|
||
match empty_result {
|
||
Ok(document) => {
|
||
println!("✅ Empty file uploaded: {}", document.id);
|
||
|
||
// Try to process empty file
|
||
match client.wait_for_processing(&document.id.to_string()).await {
|
||
Ok(processed) => {
|
||
println!("✅ Empty file processing completed: {:?}", processed.ocr_status);
|
||
}
|
||
Err(e) => {
|
||
println!("ℹ️ Empty file processing failed as expected: {}", e);
|
||
}
|
||
}
|
||
}
|
||
Err(e) => {
|
||
println!("ℹ️ Empty file upload rejected as expected: {}", e);
|
||
}
|
||
}
|
||
|
||
// Test 2: Very large text content
|
||
println!("🔍 Testing large file processing...");
|
||
|
||
let large_content = "Large file test content. ".repeat(10000);
|
||
let large_result = client.upload_file(&large_content, "large.txt", "text/plain").await;
|
||
|
||
match large_result {
|
||
Ok(document) => {
|
||
println!("✅ Large file uploaded: {} (size: {} bytes)", document.id, document.file_size);
|
||
|
||
// Give more time for large file processing
|
||
let start = Instant::now();
|
||
let extended_timeout = Duration::from_secs(180);
|
||
|
||
while start.elapsed() < extended_timeout {
|
||
let response = client.client
|
||
.get(&format!("{}/api/documents", get_base_url()))
|
||
.header("Authorization", format!("Bearer {}", client.token.as_ref().unwrap()))
|
||
.send()
|
||
.await;
|
||
|
||
if let Ok(resp) = response {
|
||
if let Ok(paginated_response) = resp.json::<PaginatedDocumentsResponse>().await {
|
||
let docs = paginated_response.documents;
|
||
if let Some(doc) = docs.iter().find(|d| d.id.to_string() == document.id.to_string()) {
|
||
match doc.ocr_status.as_deref() {
|
||
Some("completed") => {
|
||
println!("✅ Large file processing completed");
|
||
break;
|
||
}
|
||
Some("failed") => {
|
||
println!("ℹ️ Large file processing failed (may be expected for very large files)");
|
||
break;
|
||
}
|
||
_ => {
|
||
sleep(Duration::from_secs(2)).await;
|
||
continue;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
sleep(Duration::from_secs(2)).await;
|
||
}
|
||
}
|
||
Err(e) => {
|
||
println!("ℹ️ Large file upload failed (may be expected): {}", e);
|
||
}
|
||
}
|
||
|
||
// Test 3: Invalid file content but valid MIME type
|
||
println!("🔍 Testing corrupted file processing...");
|
||
|
||
let corrupted_content = "This is not actually a PDF file content";
|
||
let corrupted_result = client.upload_file(corrupted_content, "fake.pdf", "application/pdf").await;
|
||
|
||
match corrupted_result {
|
||
Ok(document) => {
|
||
println!("✅ Corrupted file uploaded: {}", document.id);
|
||
|
||
// Processing should handle the mismatch gracefully
|
||
match client.wait_for_processing(&document.id.to_string()).await {
|
||
Ok(processed) => {
|
||
println!("✅ Corrupted file processed: {:?}", processed.ocr_status);
|
||
}
|
||
Err(e) => {
|
||
println!("ℹ️ Corrupted file processing failed as expected: {}", e);
|
||
}
|
||
}
|
||
}
|
||
Err(e) => {
|
||
println!("ℹ️ Corrupted file upload handled: {}", e);
|
||
}
|
||
}
|
||
|
||
// Test 4: Special characters in filename
|
||
println!("🔍 Testing special characters in filename...");
|
||
|
||
let special_filename = "test file with spaces & special chars!@#$%^&*()_+.txt";
|
||
let special_result = client.upload_file("Content with special filename", special_filename, "text/plain").await;
|
||
|
||
match special_result {
|
||
Ok(document) => {
|
||
println!("✅ File with special characters uploaded: {}", document.id);
|
||
println!("✅ Filename preserved: {}", document.filename);
|
||
|
||
match client.wait_for_processing(&document.id.to_string()).await {
|
||
Ok(_) => println!("✅ Special filename file processed successfully"),
|
||
Err(e) => println!("⚠️ Special filename file processing failed: {}", e),
|
||
}
|
||
}
|
||
Err(e) => {
|
||
println!("ℹ️ Special filename upload handled: {}", e);
|
||
}
|
||
}
|
||
|
||
println!("🎉 Processing error recovery test completed!");
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_pipeline_performance_monitoring() {
|
||
println!("📊 Testing pipeline performance monitoring...");
|
||
|
||
let mut client = FileProcessingTestClient::new();
|
||
client.setup_user().await
|
||
.expect("Failed to setup test user");
|
||
|
||
println!("✅ User setup complete");
|
||
|
||
// Upload multiple files to test pipeline performance
|
||
let test_files = vec![
|
||
("Short text".to_string(), "short.txt"),
|
||
("Medium length text content for performance testing. ".repeat(50), "medium.txt"),
|
||
("Long text content for performance testing. ".repeat(500), "long.txt"),
|
||
];
|
||
|
||
let mut performance_results = Vec::new();
|
||
|
||
for (content, filename) in &test_files {
|
||
println!("📤 Testing performance for {}...", filename);
|
||
|
||
let upload_start = Instant::now();
|
||
|
||
let document = client.upload_file(content, filename, "text/plain").await
|
||
.expect("Failed to upload file for performance test");
|
||
|
||
let upload_time = upload_start.elapsed();
|
||
let processing_start = Instant::now();
|
||
|
||
println!("✅ {} uploaded in {:?}", filename, upload_time);
|
||
|
||
// Wait for processing and measure time
|
||
match client.wait_for_processing(&document.id.to_string()).await {
|
||
Ok(processed_doc) => {
|
||
let total_processing_time = processing_start.elapsed();
|
||
|
||
// Get OCR results to check reported processing time
|
||
if let Ok(ocr_results) = client.get_ocr_results(&document.id.to_string()).await {
|
||
let reported_time = ocr_results["ocr_processing_time_ms"]
|
||
.as_i64()
|
||
.map(|ms| Duration::from_millis(ms as u64));
|
||
|
||
performance_results.push((
|
||
filename.to_string(),
|
||
content.len(),
|
||
upload_time,
|
||
total_processing_time,
|
||
reported_time,
|
||
processed_doc.ocr_status.clone(),
|
||
));
|
||
|
||
println!("✅ {} processed in {:?} (reported: {:?})",
|
||
filename, total_processing_time, reported_time);
|
||
} else {
|
||
performance_results.push((
|
||
filename.to_string(),
|
||
content.len(),
|
||
upload_time,
|
||
total_processing_time,
|
||
None,
|
||
processed_doc.ocr_status.clone(),
|
||
));
|
||
}
|
||
}
|
||
Err(e) => {
|
||
println!("⚠️ {} processing failed: {}", filename, e);
|
||
performance_results.push((
|
||
filename.to_string(),
|
||
content.len(),
|
||
upload_time,
|
||
Duration::ZERO,
|
||
None,
|
||
Some("failed".to_string()),
|
||
));
|
||
}
|
||
}
|
||
}
|
||
|
||
// Analyze performance results
|
||
println!("📊 Performance Analysis:");
|
||
println!(" {:<12} {:<8} {:<10} {:<12} {:<10} {}", "File", "Size", "Upload", "Processing", "Reported", "Status");
|
||
println!(" {}", "-".repeat(70));
|
||
|
||
for (filename, size, upload_time, processing_time, reported_time, status) in &performance_results {
|
||
let reported_str = reported_time
|
||
.map(|d| format!("{:?}", d))
|
||
.unwrap_or_else(|| "N/A".to_string());
|
||
|
||
let status_str = status.as_deref().unwrap_or("unknown");
|
||
|
||
println!(" {:<12} {:<8} {:<10?} {:<12?} {:<10} {}",
|
||
filename, size, upload_time, processing_time, reported_str, status_str);
|
||
}
|
||
|
||
// Performance assertions
|
||
let successful_results: Vec<_> = performance_results.iter()
|
||
.filter(|(_, _, _, _, _, status)| status.as_deref() == Some("completed"))
|
||
.collect();
|
||
|
||
assert!(!successful_results.is_empty(), "At least some files should process successfully");
|
||
|
||
// Check that processing time generally correlates with file size
|
||
if successful_results.len() > 1 {
|
||
let avg_processing_time: Duration = successful_results.iter()
|
||
.map(|(_, _, _, processing_time, _, _)| *processing_time)
|
||
.sum::<Duration>() / successful_results.len() as u32;
|
||
|
||
println!("✅ Average processing time: {:?}", avg_processing_time);
|
||
|
||
// Processing should be reasonable (under 30 seconds for test files)
|
||
assert!(avg_processing_time < Duration::from_secs(30), "Average processing time should be reasonable");
|
||
}
|
||
|
||
println!("🎉 Pipeline performance monitoring test passed!");
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_concurrent_file_processing() {
|
||
println!("🔄 Testing concurrent file processing...");
|
||
|
||
let mut client = FileProcessingTestClient::new();
|
||
client.setup_user().await
|
||
.expect("Failed to setup test user");
|
||
|
||
println!("✅ User setup complete");
|
||
|
||
// Upload multiple files concurrently
|
||
let concurrent_count = 5;
|
||
let mut upload_handles = Vec::new();
|
||
|
||
for i in 0..concurrent_count {
|
||
let content = format!("Concurrent processing test document {}.\n\
|
||
This document is being processed alongside {} other documents.\n\
|
||
The system should handle multiple files efficiently.\n\
|
||
Document UUID: {}",
|
||
i + 1, concurrent_count - 1, Uuid::new_v4());
|
||
let filename = format!("concurrent_{}.txt", i + 1);
|
||
|
||
// Create a client for this upload
|
||
let token = client.token.clone().unwrap();
|
||
let client_clone = client.client.clone();
|
||
|
||
let handle = tokio::spawn(async move {
|
||
let part = reqwest::multipart::Part::text(content)
|
||
.file_name(filename.clone())
|
||
.mime_str("text/plain")
|
||
.expect("Failed to create multipart");
|
||
let form = reqwest::multipart::Form::new()
|
||
.part("file", part);
|
||
|
||
let start = Instant::now();
|
||
let response = client_clone
|
||
.post(&format!("{}/api/documents", get_base_url()))
|
||
.header("Authorization", format!("Bearer {}", token))
|
||
.multipart(form)
|
||
.send()
|
||
.await
|
||
.expect("Upload should complete");
|
||
|
||
let upload_time = start.elapsed();
|
||
|
||
if response.status().is_success() {
|
||
let response_text = response.text().await
|
||
.expect("Should get response text");
|
||
let document: DocumentUploadResponse = serde_json::from_str(&response_text)
|
||
.expect("Should parse document upload response");
|
||
Ok((i, document, upload_time))
|
||
} else {
|
||
Err((i, response.text().await.unwrap_or_default()))
|
||
}
|
||
});
|
||
|
||
upload_handles.push(handle);
|
||
}
|
||
|
||
// Wait for all uploads to complete
|
||
let mut uploaded_documents = Vec::new();
|
||
for handle in upload_handles {
|
||
match handle.await.expect("Upload task should complete") {
|
||
Ok((index, document, upload_time)) => {
|
||
println!("✅ Document {} uploaded in {:?}: {}", index + 1, upload_time, document.id);
|
||
uploaded_documents.push(document);
|
||
}
|
||
Err((index, error)) => {
|
||
println!("⚠️ Document {} upload failed: {}", index + 1, error);
|
||
}
|
||
}
|
||
}
|
||
|
||
assert!(!uploaded_documents.is_empty(), "At least some uploads should succeed");
|
||
println!("✅ {} files uploaded concurrently", uploaded_documents.len());
|
||
|
||
// Now wait for all processing to complete
|
||
let mut processing_handles: Vec<tokio::task::JoinHandle<Result<(String, Duration, &str), Box<dyn std::error::Error + Send + Sync>>>> = Vec::new();
|
||
|
||
for document in uploaded_documents {
|
||
let token = client.token.clone().unwrap();
|
||
let client_clone = client.client.clone();
|
||
let document_id = document.id.to_string();
|
||
|
||
let handle = tokio::spawn(async move {
|
||
let start = Instant::now();
|
||
|
||
// Wait for processing with timeout
|
||
while start.elapsed() < PROCESSING_TIMEOUT {
|
||
let response = client_clone
|
||
.get(&format!("{}/api/documents", get_base_url()))
|
||
.header("Authorization", format!("Bearer {}", token))
|
||
.send()
|
||
.await
|
||
.expect("Should get documents");
|
||
|
||
if response.status().is_success() {
|
||
let paginated_response: PaginatedDocumentsResponse = response.json().await
|
||
.expect("Should parse response");
|
||
let documents = paginated_response.documents;
|
||
|
||
if let Some(doc) = documents.iter().find(|d| d.id.to_string() == document_id) {
|
||
match doc.ocr_status.as_deref() {
|
||
Some("completed") => {
|
||
return Ok((document_id, start.elapsed(), "completed"));
|
||
}
|
||
Some("failed") => {
|
||
return Ok((document_id, start.elapsed(), "failed"));
|
||
}
|
||
_ => {
|
||
sleep(Duration::from_millis(1000)).await;
|
||
continue;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
sleep(Duration::from_millis(1000)).await;
|
||
}
|
||
|
||
Ok((document_id, start.elapsed(), "timeout"))
|
||
});
|
||
|
||
processing_handles.push(handle);
|
||
}
|
||
|
||
// Collect processing results
|
||
let mut processing_results = Vec::new();
|
||
for handle in processing_handles {
|
||
match handle.await.expect("Processing task should complete") {
|
||
Ok((doc_id, duration, status)) => {
|
||
println!("✅ Document {} processing {}: {:?}", doc_id, status, duration);
|
||
processing_results.push((doc_id, duration, status));
|
||
}
|
||
Err(e) => {
|
||
println!("⚠️ Processing task failed: {:?}", e);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Analyze concurrent processing results
|
||
let completed_count = processing_results.iter()
|
||
.filter(|(_, _, status)| *status == "completed")
|
||
.count();
|
||
|
||
let failed_count = processing_results.iter()
|
||
.filter(|(_, _, status)| *status == "failed")
|
||
.count();
|
||
|
||
let timeout_count = processing_results.iter()
|
||
.filter(|(_, _, status)| *status == "timeout")
|
||
.count();
|
||
|
||
println!("📊 Concurrent Processing Results:");
|
||
println!(" Completed: {}", completed_count);
|
||
println!(" Failed: {}", failed_count);
|
||
println!(" Timeout: {}", timeout_count);
|
||
|
||
if completed_count > 0 {
|
||
let avg_processing_time: Duration = processing_results.iter()
|
||
.filter(|(_, _, status)| *status == "completed")
|
||
.map(|(_, duration, _)| *duration)
|
||
.sum::<Duration>() / completed_count as u32;
|
||
|
||
println!(" Average processing time: {:?}", avg_processing_time);
|
||
}
|
||
|
||
// At least some files should process successfully
|
||
assert!(completed_count > 0, "At least some files should process successfully under concurrent load");
|
||
|
||
// Most files should not timeout (indicates system responsiveness)
|
||
let success_rate = (completed_count + failed_count) as f64 / processing_results.len() as f64;
|
||
assert!(success_rate >= 0.8, "At least 80% of files should complete processing (not timeout)");
|
||
|
||
println!("🎉 Concurrent file processing test passed!");
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_real_test_images_processing() {
|
||
println!("🖼️ Testing real test images processing...");
|
||
|
||
// Check if test images are available (simplified check)
|
||
// if !readur::test_utils::test_images_available() {
|
||
// println!("⚠️ Test images not available - skipping real image processing test");
|
||
// return;
|
||
// }
|
||
|
||
let mut client = FileProcessingTestClient::new();
|
||
client.setup_user().await
|
||
.expect("Failed to setup test user");
|
||
|
||
println!("✅ User setup complete");
|
||
|
||
// let available_images = readur::test_utils::get_available_test_images();
|
||
let available_images: Vec<TestImage> = vec![];
|
||
|
||
if available_images.is_empty() {
|
||
println!("⚠️ No test images found - skipping test");
|
||
return;
|
||
}
|
||
|
||
println!("📋 Found {} test images to process", available_images.len());
|
||
|
||
let mut processed_results = Vec::new();
|
||
|
||
// Process each available test image
|
||
for test_image in available_images.iter().take(3) { // Limit to first 3 for faster testing
|
||
println!("📤 Processing test image: {}", test_image.filename);
|
||
|
||
// Load the image data
|
||
let image_data = match test_image.load_data() {
|
||
Ok(data) => data,
|
||
Err(e) => {
|
||
println!("⚠️ Failed to load {}: {}", test_image.filename, e);
|
||
continue;
|
||
}
|
||
};
|
||
|
||
println!("✅ Loaded {} ({} bytes, {})",
|
||
test_image.filename, image_data.len(), test_image.mime_type);
|
||
|
||
// Upload the image
|
||
let upload_start = std::time::Instant::now();
|
||
let document = match client.upload_binary_file(
|
||
image_data,
|
||
&test_image.filename,
|
||
&test_image.mime_type
|
||
).await {
|
||
Ok(doc) => doc,
|
||
Err(e) => {
|
||
println!("⚠️ Failed to upload {}: {}", test_image.filename, e);
|
||
continue;
|
||
}
|
||
};
|
||
|
||
let upload_time = upload_start.elapsed();
|
||
println!("✅ {} uploaded in {:?}: {}", test_image.filename, upload_time, document.id);
|
||
|
||
// Wait for OCR processing
|
||
let processing_start = std::time::Instant::now();
|
||
match client.wait_for_processing(&document.id.to_string()).await {
|
||
Ok(processed_doc) => {
|
||
let processing_time = processing_start.elapsed();
|
||
println!("✅ {} processed in {:?}: status = {:?}",
|
||
test_image.filename, processing_time, processed_doc.ocr_status);
|
||
|
||
// Get OCR results and verify content
|
||
if let Ok(ocr_results) = client.get_ocr_results(&document.id.to_string()).await {
|
||
if let Some(ocr_text) = ocr_results["ocr_text"].as_str() {
|
||
let normalized_ocr = ocr_text.trim().to_lowercase();
|
||
let normalized_expected = test_image.expected_content.as_ref().map(|s| s.trim().to_lowercase()).unwrap_or_default();
|
||
|
||
println!("🔍 OCR extracted: '{}'", ocr_text);
|
||
println!("🎯 Expected: '{}'", test_image.expected_content.as_ref().unwrap_or(&"None".to_string()));
|
||
|
||
// Check if OCR content matches expectations
|
||
let test_number = test_image.filename.chars()
|
||
.filter(|c| c.is_numeric())
|
||
.collect::<String>();
|
||
|
||
let content_matches = if !test_number.is_empty() {
|
||
normalized_ocr.contains(&format!("test {}", test_number)) ||
|
||
normalized_ocr.contains(&test_number)
|
||
} else {
|
||
false
|
||
};
|
||
|
||
let has_text_content = normalized_ocr.contains("text") ||
|
||
normalized_ocr.contains("some");
|
||
|
||
processed_results.push((
|
||
test_image.filename.to_string(),
|
||
upload_time,
|
||
processing_time,
|
||
processed_doc.ocr_status.clone(),
|
||
ocr_text.to_string(),
|
||
content_matches,
|
||
has_text_content,
|
||
));
|
||
|
||
if content_matches && has_text_content {
|
||
println!("✅ OCR content verification PASSED for {}", test_image.filename);
|
||
} else {
|
||
println!("⚠️ OCR content verification PARTIAL for {} (number: {}, text: {})",
|
||
test_image.filename, content_matches, has_text_content);
|
||
}
|
||
} else {
|
||
println!("⚠️ No OCR text found for {}", test_image.filename);
|
||
processed_results.push((
|
||
test_image.filename.to_string(),
|
||
upload_time,
|
||
processing_time,
|
||
processed_doc.ocr_status.clone(),
|
||
"".to_string(),
|
||
false,
|
||
false,
|
||
));
|
||
}
|
||
} else {
|
||
println!("⚠️ Failed to get OCR results for {}", test_image.filename);
|
||
processed_results.push((
|
||
test_image.filename.to_string(),
|
||
upload_time,
|
||
processing_time,
|
||
processed_doc.ocr_status.clone(),
|
||
"".to_string(),
|
||
false,
|
||
false,
|
||
));
|
||
}
|
||
}
|
||
Err(e) => {
|
||
println!("⚠️ Processing failed for {}: {}", test_image.filename, e);
|
||
processed_results.push((
|
||
test_image.filename.to_string(),
|
||
upload_time,
|
||
Duration::ZERO,
|
||
Some("failed".to_string()),
|
||
"".to_string(),
|
||
false,
|
||
false,
|
||
));
|
||
}
|
||
}
|
||
|
||
// Add small delay between uploads to avoid overwhelming the system
|
||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||
}
|
||
|
||
// Analyze results
|
||
println!("📊 Real Test Images Processing Results:");
|
||
println!(" {:<12} {:<10} {:<12} {:<10} {:<8} {:<8} {}",
|
||
"Image", "Upload", "Processing", "Status", "Number", "Text", "OCR Content");
|
||
println!(" {}", "-".repeat(80));
|
||
|
||
let mut successful_ocr = 0;
|
||
let mut failed_ocr = 0;
|
||
let mut partial_matches = 0;
|
||
|
||
for (filename, upload_time, processing_time, status, ocr_text, number_match, text_match) in &processed_results {
|
||
let status_str = status.as_deref().unwrap_or("unknown");
|
||
let ocr_preview = if ocr_text.len() > 30 {
|
||
format!("{}...", &ocr_text[..30])
|
||
} else {
|
||
ocr_text.clone()
|
||
};
|
||
|
||
println!(" {:<12} {:<10?} {:<12?} {:<10} {:<8} {:<8} {}",
|
||
filename, upload_time, processing_time, status_str,
|
||
if *number_match { "✅" } else { "❌" },
|
||
if *text_match { "✅" } else { "❌" },
|
||
ocr_preview);
|
||
|
||
if status_str == "completed" {
|
||
if *number_match && *text_match {
|
||
successful_ocr += 1;
|
||
} else if *number_match || *text_match {
|
||
partial_matches += 1;
|
||
} else {
|
||
failed_ocr += 1;
|
||
}
|
||
}
|
||
}
|
||
|
||
let total_processed = processed_results.len();
|
||
|
||
println!("\n📈 Summary:");
|
||
println!(" Total processed: {}", total_processed);
|
||
println!(" Successful OCR: {}", successful_ocr);
|
||
println!(" Partial matches: {}", partial_matches);
|
||
println!(" Failed OCR: {}", failed_ocr);
|
||
|
||
if total_processed > 0 {
|
||
let success_rate = (successful_ocr + partial_matches) as f64 / total_processed as f64 * 100.0;
|
||
println!(" Success rate: {:.1}%", success_rate);
|
||
|
||
// Calculate average processing time for successful cases
|
||
let successful_processing_times: Vec<_> = processed_results.iter()
|
||
.filter(|(_, _, _, status, _, number, text)| {
|
||
status.as_deref() == Some("completed") && (*number || *text)
|
||
})
|
||
.map(|(_, _, processing_time, _, _, _, _)| *processing_time)
|
||
.collect();
|
||
|
||
if !successful_processing_times.is_empty() {
|
||
let avg_processing_time = successful_processing_times.iter().sum::<Duration>()
|
||
/ successful_processing_times.len() as u32;
|
||
println!(" Average processing time: {:?}", avg_processing_time);
|
||
}
|
||
}
|
||
|
||
// Test assertions
|
||
assert!(!processed_results.is_empty(), "At least some test images should be processed");
|
||
|
||
// At least 50% should have some level of OCR success (either partial or full)
|
||
let success_count = successful_ocr + partial_matches;
|
||
assert!(success_count > 0, "At least some test images should have successful OCR");
|
||
|
||
if total_processed >= 2 {
|
||
let min_success_rate = 0.5; // 50% minimum success rate
|
||
let actual_success_rate = success_count as f64 / total_processed as f64;
|
||
assert!(actual_success_rate >= min_success_rate,
|
||
"OCR success rate should be at least {}% but was {:.1}%",
|
||
min_success_rate * 100.0, actual_success_rate * 100.0);
|
||
}
|
||
|
||
println!("🎉 Real test images processing test completed!");
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_multi_language_document_upload() {
|
||
println!("🌐 Testing multi-language document upload...");
|
||
|
||
let mut client = FileProcessingTestClient::new();
|
||
client.setup_user().await.expect("Authentication failed");
|
||
|
||
// Test content with English and Spanish text
|
||
let mixed_content = "Hello world. This is a test document. Hola mundo. Este es un documento de prueba.";
|
||
|
||
// Upload with multiple languages
|
||
let languages = &["eng", "spa"];
|
||
let document = client.upload_file_with_languages(
|
||
mixed_content,
|
||
"mixed_language_test.txt",
|
||
"text/plain",
|
||
languages
|
||
).await.expect("Multi-language upload failed");
|
||
|
||
println!("✅ Multi-language document uploaded: {}", document.id);
|
||
|
||
// Wait for processing
|
||
let processed_doc = client.wait_for_processing(&document.id.to_string()).await
|
||
.expect("Processing failed");
|
||
|
||
println!("✅ Multi-language document processed: status = {:?}", processed_doc.ocr_status);
|
||
|
||
// Verify the document has the expected status
|
||
assert_eq!(processed_doc.ocr_status.as_deref(), Some("completed"));
|
||
|
||
// Get OCR results and verify content includes both languages
|
||
let ocr_results = client.get_ocr_results(&document.id.to_string()).await
|
||
.expect("Failed to get OCR results");
|
||
|
||
if let Some(ocr_text) = ocr_results["ocr_text"].as_str() {
|
||
println!("🔍 OCR extracted: '{}'", ocr_text);
|
||
|
||
// Verify both English and Spanish content is recognized
|
||
let normalized_ocr = ocr_text.to_lowercase();
|
||
assert!(normalized_ocr.contains("hello"), "Should contain English text");
|
||
assert!(normalized_ocr.contains("hola"), "Should contain Spanish text");
|
||
|
||
println!("✅ Multi-language OCR verification PASSED");
|
||
} else {
|
||
panic!("No OCR text found for multi-language document");
|
||
}
|
||
|
||
println!("🎉 Multi-language document upload test completed!");
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_multi_language_upload_validation() {
|
||
println!("🔍 Testing multi-language upload validation...");
|
||
|
||
let mut client = FileProcessingTestClient::new();
|
||
client.setup_user().await.expect("Authentication failed");
|
||
|
||
let test_content_max = "Test document for validation - max languages";
|
||
let test_content_too_many = "Test document for validation - too many languages";
|
||
let test_content_single = "Test document for validation - single language";
|
||
|
||
// Test with available languages (we only use 2 to avoid validation errors for unavailable languages)
|
||
let max_languages = &["eng", "spa"];
|
||
let document = client.upload_file_with_languages(
|
||
test_content_max,
|
||
"max_languages_test.txt",
|
||
"text/plain",
|
||
max_languages
|
||
).await.expect("Max languages upload should succeed");
|
||
|
||
println!("✅ Max languages document uploaded: {}", document.id);
|
||
|
||
// Test with too many languages (5) - this should fail at the API level
|
||
// We simulate this by providing 5 available languages (repeating eng and spa)
|
||
let too_many_languages = &["eng", "spa", "eng", "spa", "eng"];
|
||
let upload_result = client.upload_file_with_languages(
|
||
test_content_too_many,
|
||
"too_many_languages_test.txt",
|
||
"text/plain",
|
||
too_many_languages
|
||
).await;
|
||
|
||
// This should either fail or succeed with API validation
|
||
match upload_result {
|
||
Ok(document) => {
|
||
println!("⚠️ Too many languages upload succeeded (API allows it): {}", document.id);
|
||
// If it succeeds, the API is allowing it - that's a valid implementation choice
|
||
}
|
||
Err(e) => {
|
||
println!("✅ Too many languages upload failed as expected: {}", e);
|
||
// This is the expected behavior if API validates language count
|
||
}
|
||
}
|
||
|
||
// Test with single language for comparison
|
||
let single_language = &["eng"];
|
||
let single_doc = client.upload_file_with_languages(
|
||
test_content_single,
|
||
"single_language_test.txt",
|
||
"text/plain",
|
||
single_language
|
||
).await.expect("Single language upload should succeed");
|
||
|
||
println!("✅ Single language document uploaded: {}", single_doc.id);
|
||
|
||
println!("🎉 Multi-language upload validation test completed!");
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_multi_language_binary_upload() {
|
||
println!("🖼️ Testing multi-language binary file upload...");
|
||
|
||
let mut client = FileProcessingTestClient::new();
|
||
client.setup_user().await.expect("Authentication failed");
|
||
|
||
// Create mock binary content (simulate an image with text in multiple languages)
|
||
let binary_content = b"Mock binary image data with embedded text in multiple languages".to_vec();
|
||
|
||
// Upload binary file with multiple languages (only use available languages)
|
||
let languages = &["eng", "spa"];
|
||
let document = client.upload_binary_file_with_languages(
|
||
binary_content,
|
||
"multilang_image.png",
|
||
"image/png",
|
||
languages
|
||
).await.expect("Multi-language binary upload failed");
|
||
|
||
println!("✅ Multi-language binary document uploaded: {}", document.id);
|
||
|
||
// Wait for processing - expect failure for fake image data but success for upload
|
||
let processing_result = client.wait_for_processing(&document.id.to_string()).await;
|
||
|
||
match processing_result {
|
||
Ok(processed_doc) => {
|
||
println!("✅ Multi-language binary document processed: status = {:?}", processed_doc.ocr_status);
|
||
assert!(processed_doc.ocr_status.is_some(), "OCR status should be set");
|
||
}
|
||
Err(e) => {
|
||
println!("ℹ️ Multi-language binary document OCR failed as expected for fake image data: {}", e);
|
||
// Verify the document still exists and has failed status by checking directly
|
||
let documents = client.get_documents().await.expect("Failed to get documents");
|
||
let uploaded_doc = documents.iter().find(|d| d.id == document.id).expect("Uploaded document not found");
|
||
assert_eq!(uploaded_doc.ocr_status.as_deref(), Some("failed"), "OCR status should be 'failed' for fake image data");
|
||
}
|
||
}
|
||
|
||
println!("🎉 Multi-language binary upload test completed!");
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_backwards_compatibility_single_language() {
|
||
println!("🔄 Testing backwards compatibility with single language uploads...");
|
||
|
||
let mut client = FileProcessingTestClient::new();
|
||
client.setup_user().await.expect("Authentication failed");
|
||
|
||
let traditional_content = "Test document for backwards compatibility - traditional upload";
|
||
let multi_lang_content = "Test document for backwards compatibility - multi-language upload";
|
||
|
||
// Test traditional single language upload (without multi-language parameters)
|
||
let document = client.upload_file(
|
||
traditional_content,
|
||
"backwards_compat_test.txt",
|
||
"text/plain"
|
||
).await.expect("Traditional upload failed");
|
||
|
||
println!("✅ Traditional single language document uploaded: {}", document.id);
|
||
|
||
// Test single language using multi-language method
|
||
let languages = &["eng"];
|
||
let multi_doc = client.upload_file_with_languages(
|
||
multi_lang_content,
|
||
"single_via_multi_test.txt",
|
||
"text/plain",
|
||
languages
|
||
).await.expect("Single language via multi-language method failed");
|
||
|
||
println!("✅ Single language via multi-language method uploaded: {}", multi_doc.id);
|
||
|
||
// Both should process successfully
|
||
let traditional_processed = client.wait_for_processing(&document.id.to_string()).await
|
||
.expect("Traditional processing failed");
|
||
let multi_processed = client.wait_for_processing(&multi_doc.id.to_string()).await
|
||
.expect("Multi-method processing failed");
|
||
|
||
println!("✅ Traditional processed: status = {:?}", traditional_processed.ocr_status);
|
||
println!("✅ Multi-method processed: status = {:?}", multi_processed.ocr_status);
|
||
|
||
// Both should have completed status
|
||
assert_eq!(traditional_processed.ocr_status.as_deref(), Some("completed"));
|
||
assert_eq!(multi_processed.ocr_status.as_deref(), Some("completed"));
|
||
|
||
println!("🎉 Backwards compatibility test completed!");
|
||
} |