diff --git a/frontend/src/pages/SourcesPage.tsx b/frontend/src/pages/SourcesPage.tsx index 2621a74..172b4e2 100644 --- a/frontend/src/pages/SourcesPage.tsx +++ b/frontend/src/pages/SourcesPage.tsx @@ -65,6 +65,8 @@ import { Storage as ServerIcon, Pause as PauseIcon, PlayArrow as ResumeIcon, + TextSnippet as DocumentIcon, + Visibility as OcrIcon, } from '@mui/icons-material'; import { useNavigate } from 'react-router-dom'; import api, { queueService } from '../services/api'; @@ -84,6 +86,8 @@ interface Source { total_files_synced: number; total_files_pending: number; total_size_bytes: number; + total_documents: number; + total_documents_ocr: number; created_at: string; updated_at: string; } @@ -699,7 +703,7 @@ const SourcesPage: React.FC = () => { {source.name} - + { fontWeight: 600, }} /> + } + label={`${source.total_documents} docs`} + size="small" + sx={{ + borderRadius: 2, + bgcolor: alpha(theme.palette.info.main, 0.1), + color: theme.palette.info.main, + border: `1px solid ${alpha(theme.palette.info.main, 0.3)}`, + fontSize: '0.75rem', + fontWeight: 600, + }} + /> + } + label={`${source.total_documents_ocr} OCR'd`} + size="small" + sx={{ + borderRadius: 2, + bgcolor: alpha(theme.palette.success.main, 0.1), + color: theme.palette.success.main, + border: `1px solid ${alpha(theme.palette.success.main, 0.3)}`, + fontSize: '0.75rem', + fontWeight: 600, + }} + /> {!source.enabled && ( { {/* Stats Grid */} - + } - label="Files Processed" - value={source.total_files_synced} + icon={} + label="Documents Stored" + value={source.total_documents} + color="info" + tooltip="Total number of documents currently stored from this source" + /> + + + } + label="OCR Processed" + value={source.total_documents_ocr} color="success" - tooltip="Files attempted to be synced, including duplicates and skipped files" + tooltip="Number of documents that have been successfully OCR'd" /> - - } - label="Files Pending" - value={source.total_files_pending} - color="warning" - tooltip="Files discovered but not yet processed during sync" - /> - - - } - label="Total Size (Downloaded)" - value={formatBytes(source.total_size_bytes)} - color="primary" - tooltip="Total size of files successfully downloaded from this source" - /> - - + } label="Last Sync" @@ -852,6 +873,24 @@ const SourcesPage: React.FC = () => { tooltip="When this source was last synchronized" /> + + } + label="Files Pending" + value={source.total_files_pending} + color="warning" + tooltip="Files discovered but not yet processed during sync" + /> + + + } + label="Total Size" + value={formatBytes(source.total_size_bytes)} + color="primary" + tooltip="Total size of files successfully downloaded from this source" + /> + {/* Error Alert */} diff --git a/src/db/documents.rs b/src/db/documents.rs index af1de96..1427b22 100644 --- a/src/db/documents.rs +++ b/src/db/documents.rs @@ -1510,6 +1510,7 @@ impl Database { Ok(deleted_documents) } + pub async fn find_documents_by_confidence_threshold(&self, max_confidence: f32, user_id: uuid::Uuid, user_role: crate::models::UserRole) -> Result> { let documents = if user_role == crate::models::UserRole::Admin { let rows = sqlx::query( @@ -1583,5 +1584,60 @@ impl Database { }; Ok(documents) + + pub async fn count_documents_for_source(&self, source_id: Uuid) -> Result<(i64, i64)> { + let row = sqlx::query( + r#" + SELECT + COUNT(*) as total_documents, + COUNT(CASE WHEN ocr_status = 'completed' AND ocr_text IS NOT NULL THEN 1 END) as total_documents_ocr + FROM documents + WHERE source_id = $1 + "# + ) + .bind(source_id) + .fetch_one(&self.pool) + .await?; + + let total_documents: i64 = row.get("total_documents"); + let total_documents_ocr: i64 = row.get("total_documents_ocr"); + + Ok((total_documents, total_documents_ocr)) + } + + pub async fn count_documents_for_sources(&self, source_ids: &[Uuid]) -> Result> { + if source_ids.is_empty() { + return Ok(vec![]); + } + + let query = format!( + r#" + SELECT + source_id, + COUNT(*) as total_documents, + COUNT(CASE WHEN ocr_status = 'completed' AND ocr_text IS NOT NULL THEN 1 END) as total_documents_ocr + FROM documents + WHERE source_id = ANY($1) + GROUP BY source_id + "# + ); + + let rows = sqlx::query(&query) + .bind(source_ids) + .fetch_all(&self.pool) + .await?; + + let results = rows + .into_iter() + .map(|row| { + let source_id: Uuid = row.get("source_id"); + let total_documents: i64 = row.get("total_documents"); + let total_documents_ocr: i64 = row.get("total_documents_ocr"); + (source_id, total_documents, total_documents_ocr) + }) + .collect(); + + Ok(results) + } } \ No newline at end of file diff --git a/src/models.rs b/src/models.rs index 1de61fe..22cc34c 100644 --- a/src/models.rs +++ b/src/models.rs @@ -862,6 +862,12 @@ pub struct SourceResponse { pub total_size_bytes: i64, pub created_at: DateTime, pub updated_at: DateTime, + /// Total number of documents/files currently stored from this source + #[serde(default)] + pub total_documents: i64, + /// Total number of documents that have been OCR'd from this source + #[serde(default)] + pub total_documents_ocr: i64, } #[derive(Debug, Serialize, Deserialize, ToSchema)] @@ -903,6 +909,9 @@ impl From for SourceResponse { total_size_bytes: source.total_size_bytes, created_at: source.created_at, updated_at: source.updated_at, + // These will be populated separately when needed + total_documents: 0, + total_documents_ocr: 0, } } } diff --git a/src/routes/sources.rs b/src/routes/sources.rs index 3d48654..b42fd19 100644 --- a/src/routes/sources.rs +++ b/src/routes/sources.rs @@ -50,7 +50,33 @@ async fn list_sources( .await .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; - let responses: Vec = sources.into_iter().map(|s| s.into()).collect(); + // Get source IDs for batch counting + let source_ids: Vec = sources.iter().map(|s| s.id).collect(); + + // Get document counts for all sources in one query + let counts = state + .db + .count_documents_for_sources(&source_ids) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + + // Create a map for quick lookup + let count_map: std::collections::HashMap = counts + .into_iter() + .map(|(id, total, ocr)| (id, (total, ocr))) + .collect(); + + let responses: Vec = sources + .into_iter() + .map(|s| { + let (total_docs, total_ocr) = count_map.get(&s.id).copied().unwrap_or((0, 0)); + let mut response: SourceResponse = s.into(); + response.total_documents = total_docs; + response.total_documents_ocr = total_ocr; + response + }) + .collect(); + Ok(Json(responses)) } @@ -90,7 +116,12 @@ async fn create_source( StatusCode::INTERNAL_SERVER_ERROR })?; - Ok(Json(source.into())) + let mut response: SourceResponse = source.into(); + // New sources have no documents yet + response.total_documents = 0; + response.total_documents_ocr = 0; + + Ok(Json(response)) } #[utoipa::path( @@ -129,6 +160,13 @@ async fn get_source( .await .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + // Get document counts + let (total_documents, total_documents_ocr) = state + .db + .count_documents_for_source(source_id) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + // Calculate sync progress let sync_progress = if source.total_files_pending > 0 { Some( @@ -140,8 +178,12 @@ async fn get_source( None }; + let mut source_response: SourceResponse = source.into(); + source_response.total_documents = total_documents; + source_response.total_documents_ocr = total_documents_ocr; + let response = SourceWithStats { - source: source.into(), + source: source_response, recent_documents: recent_documents.into_iter().map(|d| d.into()).collect(), sync_progress, }; @@ -202,8 +244,19 @@ async fn update_source( StatusCode::INTERNAL_SERVER_ERROR })?; - info!("Successfully updated source {}: {}", source_id, source.name); - Ok(Json(source.into())) + // Get document counts + let (total_documents, total_documents_ocr) = state + .db + .count_documents_for_source(source_id) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + + let mut response: SourceResponse = source.into(); + response.total_documents = total_documents; + response.total_documents_ocr = total_documents_ocr; + + info!("Successfully updated source {}: {}", source_id, response.name); + Ok(Json(response)) } #[utoipa::path( diff --git a/tests/admin_functionality_tests.rs b/tests/integration_admin_functionality_tests.rs similarity index 100% rename from tests/admin_functionality_tests.rs rename to tests/integration_admin_functionality_tests.rs diff --git a/tests/auto_resume_tests.rs b/tests/integration_auto_resume_tests.rs similarity index 100% rename from tests/auto_resume_tests.rs rename to tests/integration_auto_resume_tests.rs diff --git a/tests/cancellation_tests.rs b/tests/integration_cancellation_tests.rs similarity index 100% rename from tests/cancellation_tests.rs rename to tests/integration_cancellation_tests.rs diff --git a/tests/comprehensive_source_management_tests.rs b/tests/integration_comprehensive_source_management_tests.rs similarity index 100% rename from tests/comprehensive_source_management_tests.rs rename to tests/integration_comprehensive_source_management_tests.rs diff --git a/tests/debug_ocr_test.rs b/tests/integration_debug_ocr_test.rs similarity index 100% rename from tests/debug_ocr_test.rs rename to tests/integration_debug_ocr_test.rs diff --git a/tests/document_deletion_integration_tests.rs b/tests/integration_document_deletion_integration_tests.rs similarity index 100% rename from tests/document_deletion_integration_tests.rs rename to tests/integration_document_deletion_integration_tests.rs diff --git a/tests/document_upload_hash_duplicate_tests.rs b/tests/integration_document_upload_hash_duplicate_tests.rs similarity index 100% rename from tests/document_upload_hash_duplicate_tests.rs rename to tests/integration_document_upload_hash_duplicate_tests.rs diff --git a/tests/error_handling_edge_cases_tests.rs b/tests/integration_error_handling_edge_cases_tests.rs similarity index 100% rename from tests/error_handling_edge_cases_tests.rs rename to tests/integration_error_handling_edge_cases_tests.rs diff --git a/tests/failed_ocr_endpoints_tests.rs b/tests/integration_failed_ocr_endpoints_tests.rs similarity index 100% rename from tests/failed_ocr_endpoints_tests.rs rename to tests/integration_failed_ocr_endpoints_tests.rs diff --git a/tests/file_processing_pipeline_tests.rs b/tests/integration_file_processing_pipeline_tests.rs similarity index 100% rename from tests/file_processing_pipeline_tests.rs rename to tests/integration_file_processing_pipeline_tests.rs diff --git a/tests/hash_duplicate_detection_tests.rs b/tests/integration_hash_duplicate_detection_tests.rs similarity index 100% rename from tests/hash_duplicate_detection_tests.rs rename to tests/integration_hash_duplicate_detection_tests.rs diff --git a/tests/ignored_files_integration_tests.rs b/tests/integration_ignored_files_integration_tests.rs similarity index 100% rename from tests/ignored_files_integration_tests.rs rename to tests/integration_ignored_files_integration_tests.rs diff --git a/tests/integration_tests.rs b/tests/integration_integration_tests.rs similarity index 100% rename from tests/integration_tests.rs rename to tests/integration_integration_tests.rs diff --git a/tests/investigate_empty_content.rs b/tests/integration_investigate_empty_content.rs similarity index 100% rename from tests/investigate_empty_content.rs rename to tests/integration_investigate_empty_content.rs diff --git a/tests/labels_integration_tests.rs b/tests/integration_labels_integration_tests.rs similarity index 100% rename from tests/labels_integration_tests.rs rename to tests/integration_labels_integration_tests.rs diff --git a/tests/local_folder_sync_tests.rs b/tests/integration_local_folder_sync_tests.rs similarity index 100% rename from tests/local_folder_sync_tests.rs rename to tests/integration_local_folder_sync_tests.rs diff --git a/tests/ocr_corruption_tests.rs b/tests/integration_ocr_corruption_tests.rs similarity index 100% rename from tests/ocr_corruption_tests.rs rename to tests/integration_ocr_corruption_tests.rs diff --git a/tests/ocr_failure_counting_tests.rs b/tests/integration_ocr_failure_counting_tests.rs similarity index 100% rename from tests/ocr_failure_counting_tests.rs rename to tests/integration_ocr_failure_counting_tests.rs diff --git a/tests/ocr_pipeline_integration_test.rs b/tests/integration_ocr_pipeline_integration_test.rs similarity index 100% rename from tests/ocr_pipeline_integration_test.rs rename to tests/integration_ocr_pipeline_integration_test.rs diff --git a/tests/ocr_queue_management_tests.rs b/tests/integration_ocr_queue_management_tests.rs similarity index 100% rename from tests/ocr_queue_management_tests.rs rename to tests/integration_ocr_queue_management_tests.rs diff --git a/tests/performance_load_tests.rs b/tests/integration_performance_load_tests.rs similarity index 100% rename from tests/performance_load_tests.rs rename to tests/integration_performance_load_tests.rs diff --git a/tests/prometheus_metrics_tests.rs b/tests/integration_prometheus_metrics_tests.rs similarity index 100% rename from tests/prometheus_metrics_tests.rs rename to tests/integration_prometheus_metrics_tests.rs diff --git a/tests/role_based_access_control_tests.rs b/tests/integration_role_based_access_control_tests.rs similarity index 100% rename from tests/role_based_access_control_tests.rs rename to tests/integration_role_based_access_control_tests.rs diff --git a/tests/s3_sync_tests.rs b/tests/integration_s3_sync_tests.rs similarity index 100% rename from tests/s3_sync_tests.rs rename to tests/integration_s3_sync_tests.rs diff --git a/tests/simple_throttling_test.rs b/tests/integration_simple_throttling_test.rs similarity index 100% rename from tests/simple_throttling_test.rs rename to tests/integration_simple_throttling_test.rs diff --git a/tests/source_scheduler_simple_tests.rs b/tests/integration_source_scheduler_simple_tests.rs similarity index 100% rename from tests/source_scheduler_simple_tests.rs rename to tests/integration_source_scheduler_simple_tests.rs diff --git a/tests/source_scheduler_tests.rs b/tests/integration_source_scheduler_tests.rs similarity index 100% rename from tests/source_scheduler_tests.rs rename to tests/integration_source_scheduler_tests.rs diff --git a/tests/source_sync_hash_duplicate_tests.rs b/tests/integration_source_sync_hash_duplicate_tests.rs similarity index 100% rename from tests/source_sync_hash_duplicate_tests.rs rename to tests/integration_source_sync_hash_duplicate_tests.rs diff --git a/tests/source_update_tests.rs b/tests/integration_source_update_tests.rs similarity index 100% rename from tests/source_update_tests.rs rename to tests/integration_source_update_tests.rs diff --git a/tests/stop_sync_functionality_tests.rs b/tests/integration_stop_sync_functionality_tests.rs similarity index 100% rename from tests/stop_sync_functionality_tests.rs rename to tests/integration_stop_sync_functionality_tests.rs diff --git a/tests/test_image_ocr_tests.rs b/tests/integration_test_image_ocr_tests.rs similarity index 100% rename from tests/test_image_ocr_tests.rs rename to tests/integration_test_image_ocr_tests.rs diff --git a/tests/thread_separation_tests.rs b/tests/integration_thread_separation_tests.rs similarity index 100% rename from tests/thread_separation_tests.rs rename to tests/integration_thread_separation_tests.rs diff --git a/tests/universal_source_sync_tests.rs b/tests/integration_universal_source_sync_tests.rs similarity index 100% rename from tests/universal_source_sync_tests.rs rename to tests/integration_universal_source_sync_tests.rs diff --git a/tests/webdav_comprehensive_tests.rs b/tests/integration_webdav_comprehensive_tests.rs similarity index 100% rename from tests/webdav_comprehensive_tests.rs rename to tests/integration_webdav_comprehensive_tests.rs diff --git a/tests/webdav_hash_duplicate_tests.rs b/tests/integration_webdav_hash_duplicate_tests.rs similarity index 100% rename from tests/webdav_hash_duplicate_tests.rs rename to tests/integration_webdav_hash_duplicate_tests.rs diff --git a/tests/webdav_integration_tests.rs b/tests/integration_webdav_integration_tests.rs similarity index 100% rename from tests/webdav_integration_tests.rs rename to tests/integration_webdav_integration_tests.rs diff --git a/tests/webdav_sync_tests.rs b/tests/integration_webdav_sync_tests.rs similarity index 100% rename from tests/webdav_sync_tests.rs rename to tests/integration_webdav_sync_tests.rs diff --git a/tests/basic_sync_tests.rs b/tests/unit_basic_sync_tests.rs similarity index 100% rename from tests/basic_sync_tests.rs rename to tests/unit_basic_sync_tests.rs diff --git a/tests/oidc_unit_tests.rs b/tests/unit_oidc_unit_tests.rs similarity index 100% rename from tests/oidc_unit_tests.rs rename to tests/unit_oidc_unit_tests.rs diff --git a/tests/unit_tests.rs b/tests/unit_unit_tests.rs similarity index 100% rename from tests/unit_tests.rs rename to tests/unit_unit_tests.rs diff --git a/tests/webdav_enhanced_unit_tests.rs b/tests/unit_webdav_enhanced_unit_tests.rs similarity index 100% rename from tests/webdav_enhanced_unit_tests.rs rename to tests/unit_webdav_enhanced_unit_tests.rs diff --git a/tests/webdav_unit_tests.rs b/tests/unit_webdav_unit_tests.rs similarity index 100% rename from tests/webdav_unit_tests.rs rename to tests/unit_webdav_unit_tests.rs