feat(client/server): add a new badge for each source that shows the number of documents stored from each source
This commit is contained in:
parent
341a91e1a7
commit
a75fca0c28
|
|
@ -65,6 +65,8 @@ import {
|
||||||
Storage as ServerIcon,
|
Storage as ServerIcon,
|
||||||
Pause as PauseIcon,
|
Pause as PauseIcon,
|
||||||
PlayArrow as ResumeIcon,
|
PlayArrow as ResumeIcon,
|
||||||
|
TextSnippet as DocumentIcon,
|
||||||
|
Visibility as OcrIcon,
|
||||||
} from '@mui/icons-material';
|
} from '@mui/icons-material';
|
||||||
import { useNavigate } from 'react-router-dom';
|
import { useNavigate } from 'react-router-dom';
|
||||||
import api, { queueService } from '../services/api';
|
import api, { queueService } from '../services/api';
|
||||||
|
|
@ -84,6 +86,8 @@ interface Source {
|
||||||
total_files_synced: number;
|
total_files_synced: number;
|
||||||
total_files_pending: number;
|
total_files_pending: number;
|
||||||
total_size_bytes: number;
|
total_size_bytes: number;
|
||||||
|
total_documents: number;
|
||||||
|
total_documents_ocr: number;
|
||||||
created_at: string;
|
created_at: string;
|
||||||
updated_at: string;
|
updated_at: string;
|
||||||
}
|
}
|
||||||
|
|
@ -699,7 +703,7 @@ const SourcesPage: React.FC = () => {
|
||||||
<Typography variant="h6" fontWeight="bold" gutterBottom>
|
<Typography variant="h6" fontWeight="bold" gutterBottom>
|
||||||
{source.name}
|
{source.name}
|
||||||
</Typography>
|
</Typography>
|
||||||
<Stack direction="row" spacing={1} alignItems="center">
|
<Stack direction="row" spacing={1} alignItems="center" flexWrap="wrap">
|
||||||
<Chip
|
<Chip
|
||||||
label={source.source_type.toUpperCase()}
|
label={source.source_type.toUpperCase()}
|
||||||
size="small"
|
size="small"
|
||||||
|
|
@ -723,6 +727,32 @@ const SourcesPage: React.FC = () => {
|
||||||
fontWeight: 600,
|
fontWeight: 600,
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
|
<Chip
|
||||||
|
icon={<DocumentIcon sx={{ fontSize: '0.9rem !important' }} />}
|
||||||
|
label={`${source.total_documents} docs`}
|
||||||
|
size="small"
|
||||||
|
sx={{
|
||||||
|
borderRadius: 2,
|
||||||
|
bgcolor: alpha(theme.palette.info.main, 0.1),
|
||||||
|
color: theme.palette.info.main,
|
||||||
|
border: `1px solid ${alpha(theme.palette.info.main, 0.3)}`,
|
||||||
|
fontSize: '0.75rem',
|
||||||
|
fontWeight: 600,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<Chip
|
||||||
|
icon={<OcrIcon sx={{ fontSize: '0.9rem !important' }} />}
|
||||||
|
label={`${source.total_documents_ocr} OCR'd`}
|
||||||
|
size="small"
|
||||||
|
sx={{
|
||||||
|
borderRadius: 2,
|
||||||
|
bgcolor: alpha(theme.palette.success.main, 0.1),
|
||||||
|
color: theme.palette.success.main,
|
||||||
|
border: `1px solid ${alpha(theme.palette.success.main, 0.3)}`,
|
||||||
|
fontSize: '0.75rem',
|
||||||
|
fontWeight: 600,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
{!source.enabled && (
|
{!source.enabled && (
|
||||||
<Chip
|
<Chip
|
||||||
label="Disabled"
|
label="Disabled"
|
||||||
|
|
@ -814,34 +844,25 @@ const SourcesPage: React.FC = () => {
|
||||||
|
|
||||||
{/* Stats Grid */}
|
{/* Stats Grid */}
|
||||||
<Grid container spacing={2} mb={3}>
|
<Grid container spacing={2} mb={3}>
|
||||||
<Grid item xs={6} sm={4} md={3}>
|
<Grid item xs={6} sm={4} md={2.4}>
|
||||||
<StatCard
|
<StatCard
|
||||||
icon={<TrendingUpIcon />}
|
icon={<DocumentIcon />}
|
||||||
label="Files Processed"
|
label="Documents Stored"
|
||||||
value={source.total_files_synced}
|
value={source.total_documents}
|
||||||
|
color="info"
|
||||||
|
tooltip="Total number of documents currently stored from this source"
|
||||||
|
/>
|
||||||
|
</Grid>
|
||||||
|
<Grid item xs={6} sm={4} md={2.4}>
|
||||||
|
<StatCard
|
||||||
|
icon={<OcrIcon />}
|
||||||
|
label="OCR Processed"
|
||||||
|
value={source.total_documents_ocr}
|
||||||
color="success"
|
color="success"
|
||||||
tooltip="Files attempted to be synced, including duplicates and skipped files"
|
tooltip="Number of documents that have been successfully OCR'd"
|
||||||
/>
|
/>
|
||||||
</Grid>
|
</Grid>
|
||||||
<Grid item xs={6} sm={4} md={3}>
|
<Grid item xs={6} sm={4} md={2.4}>
|
||||||
<StatCard
|
|
||||||
icon={<SpeedIcon />}
|
|
||||||
label="Files Pending"
|
|
||||||
value={source.total_files_pending}
|
|
||||||
color="warning"
|
|
||||||
tooltip="Files discovered but not yet processed during sync"
|
|
||||||
/>
|
|
||||||
</Grid>
|
|
||||||
<Grid item xs={6} sm={4} md={3}>
|
|
||||||
<StatCard
|
|
||||||
icon={<StorageIcon />}
|
|
||||||
label="Total Size (Downloaded)"
|
|
||||||
value={formatBytes(source.total_size_bytes)}
|
|
||||||
color="primary"
|
|
||||||
tooltip="Total size of files successfully downloaded from this source"
|
|
||||||
/>
|
|
||||||
</Grid>
|
|
||||||
<Grid item xs={6} sm={4} md={3}>
|
|
||||||
<StatCard
|
<StatCard
|
||||||
icon={<TimelineIcon />}
|
icon={<TimelineIcon />}
|
||||||
label="Last Sync"
|
label="Last Sync"
|
||||||
|
|
@ -852,6 +873,24 @@ const SourcesPage: React.FC = () => {
|
||||||
tooltip="When this source was last synchronized"
|
tooltip="When this source was last synchronized"
|
||||||
/>
|
/>
|
||||||
</Grid>
|
</Grid>
|
||||||
|
<Grid item xs={6} sm={4} md={2.4}>
|
||||||
|
<StatCard
|
||||||
|
icon={<SpeedIcon />}
|
||||||
|
label="Files Pending"
|
||||||
|
value={source.total_files_pending}
|
||||||
|
color="warning"
|
||||||
|
tooltip="Files discovered but not yet processed during sync"
|
||||||
|
/>
|
||||||
|
</Grid>
|
||||||
|
<Grid item xs={6} sm={4} md={2.4}>
|
||||||
|
<StatCard
|
||||||
|
icon={<StorageIcon />}
|
||||||
|
label="Total Size"
|
||||||
|
value={formatBytes(source.total_size_bytes)}
|
||||||
|
color="primary"
|
||||||
|
tooltip="Total size of files successfully downloaded from this source"
|
||||||
|
/>
|
||||||
|
</Grid>
|
||||||
</Grid>
|
</Grid>
|
||||||
|
|
||||||
{/* Error Alert */}
|
{/* Error Alert */}
|
||||||
|
|
|
||||||
|
|
@ -1509,4 +1509,59 @@ impl Database {
|
||||||
|
|
||||||
Ok(deleted_documents)
|
Ok(deleted_documents)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn count_documents_for_source(&self, source_id: Uuid) -> Result<(i64, i64)> {
|
||||||
|
let row = sqlx::query(
|
||||||
|
r#"
|
||||||
|
SELECT
|
||||||
|
COUNT(*) as total_documents,
|
||||||
|
COUNT(CASE WHEN ocr_status = 'completed' AND ocr_text IS NOT NULL THEN 1 END) as total_documents_ocr
|
||||||
|
FROM documents
|
||||||
|
WHERE source_id = $1
|
||||||
|
"#
|
||||||
|
)
|
||||||
|
.bind(source_id)
|
||||||
|
.fetch_one(&self.pool)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let total_documents: i64 = row.get("total_documents");
|
||||||
|
let total_documents_ocr: i64 = row.get("total_documents_ocr");
|
||||||
|
|
||||||
|
Ok((total_documents, total_documents_ocr))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn count_documents_for_sources(&self, source_ids: &[Uuid]) -> Result<Vec<(Uuid, i64, i64)>> {
|
||||||
|
if source_ids.is_empty() {
|
||||||
|
return Ok(vec![]);
|
||||||
|
}
|
||||||
|
|
||||||
|
let query = format!(
|
||||||
|
r#"
|
||||||
|
SELECT
|
||||||
|
source_id,
|
||||||
|
COUNT(*) as total_documents,
|
||||||
|
COUNT(CASE WHEN ocr_status = 'completed' AND ocr_text IS NOT NULL THEN 1 END) as total_documents_ocr
|
||||||
|
FROM documents
|
||||||
|
WHERE source_id = ANY($1)
|
||||||
|
GROUP BY source_id
|
||||||
|
"#
|
||||||
|
);
|
||||||
|
|
||||||
|
let rows = sqlx::query(&query)
|
||||||
|
.bind(source_ids)
|
||||||
|
.fetch_all(&self.pool)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let results = rows
|
||||||
|
.into_iter()
|
||||||
|
.map(|row| {
|
||||||
|
let source_id: Uuid = row.get("source_id");
|
||||||
|
let total_documents: i64 = row.get("total_documents");
|
||||||
|
let total_documents_ocr: i64 = row.get("total_documents_ocr");
|
||||||
|
(source_id, total_documents, total_documents_ocr)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Ok(results)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -862,6 +862,12 @@ pub struct SourceResponse {
|
||||||
pub total_size_bytes: i64,
|
pub total_size_bytes: i64,
|
||||||
pub created_at: DateTime<Utc>,
|
pub created_at: DateTime<Utc>,
|
||||||
pub updated_at: DateTime<Utc>,
|
pub updated_at: DateTime<Utc>,
|
||||||
|
/// Total number of documents/files currently stored from this source
|
||||||
|
#[serde(default)]
|
||||||
|
pub total_documents: i64,
|
||||||
|
/// Total number of documents that have been OCR'd from this source
|
||||||
|
#[serde(default)]
|
||||||
|
pub total_documents_ocr: i64,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, ToSchema)]
|
#[derive(Debug, Serialize, Deserialize, ToSchema)]
|
||||||
|
|
@ -903,6 +909,9 @@ impl From<Source> for SourceResponse {
|
||||||
total_size_bytes: source.total_size_bytes,
|
total_size_bytes: source.total_size_bytes,
|
||||||
created_at: source.created_at,
|
created_at: source.created_at,
|
||||||
updated_at: source.updated_at,
|
updated_at: source.updated_at,
|
||||||
|
// These will be populated separately when needed
|
||||||
|
total_documents: 0,
|
||||||
|
total_documents_ocr: 0,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -50,7 +50,33 @@ async fn list_sources(
|
||||||
.await
|
.await
|
||||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||||
|
|
||||||
let responses: Vec<SourceResponse> = sources.into_iter().map(|s| s.into()).collect();
|
// Get source IDs for batch counting
|
||||||
|
let source_ids: Vec<Uuid> = sources.iter().map(|s| s.id).collect();
|
||||||
|
|
||||||
|
// Get document counts for all sources in one query
|
||||||
|
let counts = state
|
||||||
|
.db
|
||||||
|
.count_documents_for_sources(&source_ids)
|
||||||
|
.await
|
||||||
|
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||||
|
|
||||||
|
// Create a map for quick lookup
|
||||||
|
let count_map: std::collections::HashMap<Uuid, (i64, i64)> = counts
|
||||||
|
.into_iter()
|
||||||
|
.map(|(id, total, ocr)| (id, (total, ocr)))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let responses: Vec<SourceResponse> = sources
|
||||||
|
.into_iter()
|
||||||
|
.map(|s| {
|
||||||
|
let (total_docs, total_ocr) = count_map.get(&s.id).copied().unwrap_or((0, 0));
|
||||||
|
let mut response: SourceResponse = s.into();
|
||||||
|
response.total_documents = total_docs;
|
||||||
|
response.total_documents_ocr = total_ocr;
|
||||||
|
response
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
Ok(Json(responses))
|
Ok(Json(responses))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -90,7 +116,12 @@ async fn create_source(
|
||||||
StatusCode::INTERNAL_SERVER_ERROR
|
StatusCode::INTERNAL_SERVER_ERROR
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
Ok(Json(source.into()))
|
let mut response: SourceResponse = source.into();
|
||||||
|
// New sources have no documents yet
|
||||||
|
response.total_documents = 0;
|
||||||
|
response.total_documents_ocr = 0;
|
||||||
|
|
||||||
|
Ok(Json(response))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[utoipa::path(
|
#[utoipa::path(
|
||||||
|
|
@ -129,6 +160,13 @@ async fn get_source(
|
||||||
.await
|
.await
|
||||||
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||||
|
|
||||||
|
// Get document counts
|
||||||
|
let (total_documents, total_documents_ocr) = state
|
||||||
|
.db
|
||||||
|
.count_documents_for_source(source_id)
|
||||||
|
.await
|
||||||
|
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||||
|
|
||||||
// Calculate sync progress
|
// Calculate sync progress
|
||||||
let sync_progress = if source.total_files_pending > 0 {
|
let sync_progress = if source.total_files_pending > 0 {
|
||||||
Some(
|
Some(
|
||||||
|
|
@ -140,8 +178,12 @@ async fn get_source(
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let mut source_response: SourceResponse = source.into();
|
||||||
|
source_response.total_documents = total_documents;
|
||||||
|
source_response.total_documents_ocr = total_documents_ocr;
|
||||||
|
|
||||||
let response = SourceWithStats {
|
let response = SourceWithStats {
|
||||||
source: source.into(),
|
source: source_response,
|
||||||
recent_documents: recent_documents.into_iter().map(|d| d.into()).collect(),
|
recent_documents: recent_documents.into_iter().map(|d| d.into()).collect(),
|
||||||
sync_progress,
|
sync_progress,
|
||||||
};
|
};
|
||||||
|
|
@ -202,8 +244,19 @@ async fn update_source(
|
||||||
StatusCode::INTERNAL_SERVER_ERROR
|
StatusCode::INTERNAL_SERVER_ERROR
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
info!("Successfully updated source {}: {}", source_id, source.name);
|
// Get document counts
|
||||||
Ok(Json(source.into()))
|
let (total_documents, total_documents_ocr) = state
|
||||||
|
.db
|
||||||
|
.count_documents_for_source(source_id)
|
||||||
|
.await
|
||||||
|
.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
|
||||||
|
|
||||||
|
let mut response: SourceResponse = source.into();
|
||||||
|
response.total_documents = total_documents;
|
||||||
|
response.total_documents_ocr = total_documents_ocr;
|
||||||
|
|
||||||
|
info!("Successfully updated source {}: {}", source_id, response.name);
|
||||||
|
Ok(Json(response))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[utoipa::path(
|
#[utoipa::path(
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue