import React, { useState, useEffect } from 'react'; import { Box, Container, Typography, Paper, Button, Card, CardContent, Chip, IconButton, Dialog, DialogTitle, DialogContent, DialogContentText, DialogActions, TextField, FormControl, InputLabel, Select, MenuItem, Alert, LinearProgress, Snackbar, Divider, FormControlLabel, Switch, Tooltip, CircularProgress, Fade, Stack, Avatar, Badge, useTheme, alpha, Table, TableBody, TableCell, TableContainer, TableHead, TableRow, } from '@mui/material'; import Grid from '@mui/material/GridLegacy'; import { Add as AddIcon, CloudSync as CloudSyncIcon, Error as ErrorIcon, CheckCircle as CheckCircleIcon, Edit as EditIcon, Delete as DeleteIcon, PlayArrow as PlayArrowIcon, Stop as StopIcon, Storage as StorageIcon, Cloud as CloudIcon, Speed as SpeedIcon, Timeline as TimelineIcon, TrendingUp as TrendingUpIcon, Security as SecurityIcon, AutoFixHigh as AutoFixHighIcon, Sync as SyncIcon, MoreVert as MoreVertIcon, Menu as MenuIcon, Speed as QuickSyncIcon, ManageSearch as DeepScanIcon, Folder as FolderIcon, Assessment as AssessmentIcon, Extension as ExtensionIcon, Storage as ServerIcon, Pause as PauseIcon, PlayArrow as ResumeIcon, TextSnippet as DocumentIcon, Visibility as OcrIcon, Block as BlockIcon, HealthAndSafety as HealthIcon, Warning as WarningIcon, Error as CriticalIcon, } from '@mui/icons-material'; import { useNavigate } from 'react-router-dom'; import api, { queueService, sourcesService, ErrorHelper, ErrorCodes } from '../services/api'; import { formatDistanceToNow } from 'date-fns'; import { useAuth } from '../contexts/AuthContext'; import { useTranslation } from 'react-i18next'; import SyncProgressDisplay from '../components/SyncProgress'; interface Source { id: string; name: string; source_type: 'webdav' | 'local_folder' | 's3'; enabled: boolean; config: any; status: 'idle' | 'syncing' | 'error'; last_sync_at: string | null; last_error: string | null; last_error_at: string | null; total_files_synced: number; total_files_pending: number; total_size_bytes: number; total_documents: number; total_documents_ocr: number; created_at: string; updated_at: string; // Validation fields validation_status?: string | null; last_validation_at?: string | null; validation_score?: number | null; validation_issues?: string | null; } interface SnackbarState { open: boolean; message: string; severity: 'success' | 'error' | 'warning' | 'info'; } const SourcesPage: React.FC = () => { const theme = useTheme(); const navigate = useNavigate(); const { user } = useAuth(); const { t } = useTranslation(); const [sources, setSources] = useState([]); const [loading, setLoading] = useState(true); const [ocrStatus, setOcrStatus] = useState<{ is_paused: boolean; status: string }>({ is_paused: false, status: 'running' }); const [ocrLoading, setOcrLoading] = useState(false); const [dialogOpen, setDialogOpen] = useState(false); const [editingSource, setEditingSource] = useState(null); const [deleteDialogOpen, setDeleteDialogOpen] = useState(false); const [sourceToDelete, setSourceToDelete] = useState(null); const [deleteLoading, setDeleteLoading] = useState(false); const [snackbar, setSnackbar] = useState({ open: false, message: '', severity: 'info', }); // Form state const [formData, setFormData] = useState({ name: '', source_type: 'webdav' as 'webdav' | 'local_folder' | 's3', enabled: true, // WebDAV fields server_url: '', username: '', password: '', server_type: 'generic' as 'nextcloud' | 'owncloud' | 'generic', // Local Folder fields recursive: true, follow_symlinks: false, // S3 fields bucket_name: '', region: 'us-east-1', access_key_id: '', secret_access_key: '', endpoint_url: '', prefix: '', // Common fields watch_folders: ['/Documents'], file_extensions: ['pdf', 'png', 'jpg', 'jpeg', 'tiff', 'bmp', 'txt'], auto_sync: false, sync_interval_minutes: 60, }); // Additional state for enhanced features const [newFolder, setNewFolder] = useState(''); const [newExtension, setNewExtension] = useState(''); const [crawlEstimate, setCrawlEstimate] = useState(null); const [estimatingCrawl, setEstimatingCrawl] = useState(false); const [testingConnection, setTestingConnection] = useState(false); const [syncingSource, setSyncingSource] = useState(null); const [stoppingSync, setStoppingSync] = useState(null); const [validating, setValidating] = useState(null); const [autoRefreshing, setAutoRefreshing] = useState(false); // Sync modal state const [syncModalOpen, setSyncModalOpen] = useState(false); const [sourceToSync, setSourceToSync] = useState(null); const [deepScanning, setDeepScanning] = useState(false); useEffect(() => { loadSources(); if (user?.role === 'Admin') { loadOcrStatus(); } }, [user]); // Auto-refresh sources when any source is syncing useEffect(() => { const activeSyncingSources = sources.filter(source => source.status === 'syncing'); if (activeSyncingSources.length > 0) { setAutoRefreshing(true); const interval = setInterval(() => { loadSources(); }, 5000); // Poll every 5 seconds during active sync return () => { clearInterval(interval); setAutoRefreshing(false); }; } else { setAutoRefreshing(false); } }, [sources]); // Update default folders when source type changes useEffect(() => { if (!editingSource) { // Only for new sources let defaultFolders; switch (formData.source_type) { case 'local_folder': defaultFolders = ['/home/user/Documents']; break; case 's3': defaultFolders = ['documents/']; break; case 'webdav': default: defaultFolders = ['/Documents']; break; } setFormData(prev => ({ ...prev, watch_folders: defaultFolders })); } }, [formData.source_type, editingSource]); const loadSources = async () => { try { const response = await api.get('/sources'); setSources(response.data); } catch (error) { console.error('Failed to load sources:', error); showSnackbar(t('sources.errors.loadFailed'), 'error'); } finally { setLoading(false); } }; const showSnackbar = (message: string, severity: SnackbarState['severity']) => { setSnackbar({ open: true, message, severity }); }; // OCR Control Functions (Admin only) const loadOcrStatus = async () => { if (user?.role !== 'Admin') return; try { const response = await queueService.getOcrStatus(); setOcrStatus(response.data); } catch (error) { console.error('Failed to load OCR status:', error); } }; const handlePauseOcr = async () => { if (user?.role !== 'Admin') return; setOcrLoading(true); try { await queueService.pauseOcr(); await loadOcrStatus(); showSnackbar(t('sources.ocr.pausedSuccess'), 'success'); } catch (error) { console.error('Failed to pause OCR:', error); showSnackbar(t('sources.ocr.pauseFailed'), 'error'); } finally { setOcrLoading(false); } }; const handleResumeOcr = async () => { if (user?.role !== 'Admin') return; setOcrLoading(true); try { await queueService.resumeOcr(); await loadOcrStatus(); showSnackbar(t('sources.ocr.resumedSuccess'), 'success'); } catch (error) { console.error('Failed to resume OCR:', error); showSnackbar(t('sources.ocr.resumeFailed'), 'error'); } finally { setOcrLoading(false); } }; const handleCreateSource = () => { setEditingSource(null); setFormData({ name: '', source_type: 'webdav', enabled: true, // WebDAV fields server_url: '', username: '', password: '', server_type: 'generic', // Local Folder fields recursive: true, follow_symlinks: false, // S3 fields bucket_name: '', region: 'us-east-1', access_key_id: '', secret_access_key: '', endpoint_url: '', prefix: '', // Common fields watch_folders: ['/Documents'], file_extensions: ['pdf', 'png', 'jpg', 'jpeg', 'tiff', 'bmp', 'txt'], auto_sync: false, sync_interval_minutes: 60, }); setCrawlEstimate(null); setNewFolder(''); setNewExtension(''); setDialogOpen(true); }; const handleEditSource = (source: Source) => { setEditingSource(source); const config = source.config; setFormData({ name: source.name, source_type: source.source_type, enabled: source.enabled, // WebDAV fields server_url: config.server_url || '', username: config.username || '', password: config.password || '', server_type: config.server_type || 'generic', // Local Folder fields recursive: config.recursive !== undefined ? config.recursive : true, follow_symlinks: config.follow_symlinks || false, // S3 fields bucket_name: config.bucket_name || '', region: config.region || 'us-east-1', access_key_id: config.access_key_id || '', secret_access_key: config.secret_access_key || '', endpoint_url: config.endpoint_url || '', prefix: config.prefix || '', // Common fields watch_folders: config.watch_folders || ['/Documents'], file_extensions: config.file_extensions || ['pdf', 'png', 'jpg', 'jpeg', 'tiff', 'bmp', 'txt'], auto_sync: config.auto_sync || false, sync_interval_minutes: config.sync_interval_minutes || 60, }); setCrawlEstimate(null); setNewFolder(''); setNewExtension(''); setDialogOpen(true); }; const handleSaveSource = async () => { try { let config = {}; // Build config based on source type if (formData.source_type === 'webdav') { config = { server_url: formData.server_url, username: formData.username, password: formData.password, watch_folders: formData.watch_folders, file_extensions: formData.file_extensions, auto_sync: formData.auto_sync, sync_interval_minutes: formData.sync_interval_minutes, server_type: formData.server_type, }; } else if (formData.source_type === 'local_folder') { config = { watch_folders: formData.watch_folders, file_extensions: formData.file_extensions, auto_sync: formData.auto_sync, sync_interval_minutes: formData.sync_interval_minutes, recursive: formData.recursive, follow_symlinks: formData.follow_symlinks, }; } else if (formData.source_type === 's3') { config = { bucket_name: formData.bucket_name, region: formData.region, access_key_id: formData.access_key_id, secret_access_key: formData.secret_access_key, endpoint_url: formData.endpoint_url, prefix: formData.prefix, watch_folders: formData.watch_folders, file_extensions: formData.file_extensions, auto_sync: formData.auto_sync, sync_interval_minutes: formData.sync_interval_minutes, }; } if (editingSource) { await api.put(`/sources/${editingSource.id}`, { name: formData.name, enabled: formData.enabled, config, }); showSnackbar(t('sources.messages.updateSuccess'), 'success'); } else { await api.post('/sources', { name: formData.name, source_type: formData.source_type, enabled: formData.enabled, config, }); showSnackbar(t('sources.messages.createSuccess'), 'success'); } setDialogOpen(false); loadSources(); } catch (error) { console.error('Failed to save source:', error); const errorInfo = ErrorHelper.formatErrorForDisplay(error, true); // Handle specific source errors if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_DUPLICATE_NAME)) { showSnackbar(t('sources.errors.duplicateName'), 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_CONFIG_INVALID)) { showSnackbar(t('sources.errors.invalidConfig'), 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_AUTH_FAILED)) { showSnackbar(t('sources.errors.authFailed'), 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_CONNECTION_FAILED)) { showSnackbar(t('sources.errors.connectionError'), 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_INVALID_PATH)) { showSnackbar(t('sources.errors.invalidPath'), 'error'); } else { showSnackbar(errorInfo.message || t('sources.errors.saveFailed'), 'error'); } } }; const handleDeleteSource = (source: Source) => { setSourceToDelete(source); setDeleteDialogOpen(true); }; const handleDeleteCancel = () => { setDeleteDialogOpen(false); setSourceToDelete(null); setDeleteLoading(false); }; const handleDeleteConfirm = async () => { if (!sourceToDelete) return; setDeleteLoading(true); try { await api.delete(`/sources/${sourceToDelete.id}`); showSnackbar(t('sources.messages.deleteSuccess'), 'success'); loadSources(); handleDeleteCancel(); } catch (error) { console.error('Failed to delete source:', error); const errorInfo = ErrorHelper.formatErrorForDisplay(error, true); // Handle specific delete errors if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_NOT_FOUND)) { showSnackbar(t('sources.errors.notFound'), 'warning'); loadSources(); // Refresh the list handleDeleteCancel(); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_SYNC_IN_PROGRESS)) { showSnackbar(t('sources.errors.syncInProgress'), 'error'); } else { showSnackbar(errorInfo.message || t('sources.errors.deleteFailed'), 'error'); } setDeleteLoading(false); } }; const handleTestConnection = async () => { setTestingConnection(true); try { let response; if (formData.source_type === 'webdav') { response = await api.post('/sources/test-connection', { source_type: 'webdav', config: { server_url: formData.server_url, username: formData.username, password: formData.password, server_type: formData.server_type, watch_folders: formData.watch_folders, file_extensions: formData.file_extensions, } }); } else if (formData.source_type === 'local_folder') { response = await api.post('/sources/test-connection', { source_type: 'local_folder', config: { watch_folders: formData.watch_folders, file_extensions: formData.file_extensions, recursive: formData.recursive, follow_symlinks: formData.follow_symlinks, } }); } else if (formData.source_type === 's3') { response = await api.post('/sources/test-connection', { source_type: 's3', config: { bucket_name: formData.bucket_name, region: formData.region, access_key_id: formData.access_key_id, secret_access_key: formData.secret_access_key, endpoint_url: formData.endpoint_url, prefix: formData.prefix, } }); } if (response && response.data.success) { showSnackbar(response.data.message || t('sources.messages.connectionSuccess'), 'success'); } else { showSnackbar(response?.data.message || t('sources.errors.connectionFailed'), 'error'); } } catch (error: any) { console.error('Failed to test connection:', error); const errorInfo = ErrorHelper.formatErrorForDisplay(error, true); // Handle specific connection test errors if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_CONNECTION_FAILED)) { showSnackbar(t('sources.errors.connectionFailedUrl'), 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_AUTH_FAILED)) { showSnackbar(t('sources.errors.authFailedCredentials'), 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_INVALID_PATH)) { showSnackbar(t('sources.errors.invalidFolderPath'), 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_CONFIG_INVALID)) { showSnackbar(t('sources.errors.invalidSettings'), 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_NETWORK_TIMEOUT)) { showSnackbar(t('sources.errors.timeout'), 'error'); } else { showSnackbar(errorInfo.message || t('sources.errors.testConnectionFailed'), 'error'); } } finally { setTestingConnection(false); } }; // Open sync modal instead of directly triggering sync const handleOpenSyncModal = (source: Source) => { setSourceToSync(source); setSyncModalOpen(true); }; const handleCloseSyncModal = () => { setSyncModalOpen(false); setSourceToSync(null); }; const handleQuickSync = async () => { if (!sourceToSync) return; setSyncingSource(sourceToSync.id); handleCloseSyncModal(); try { await sourcesService.triggerSync(sourceToSync.id); showSnackbar(t('sources.messages.syncStartSuccess'), 'success'); setTimeout(loadSources, 1000); } catch (error: any) { console.error('Failed to trigger sync:', error); const errorInfo = ErrorHelper.formatErrorForDisplay(error, true); // Handle specific sync errors if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_SYNC_IN_PROGRESS)) { showSnackbar(t('sources.errors.alreadySyncing'), 'warning'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_CONNECTION_FAILED)) { showSnackbar(t('sources.errors.cannotConnect'), 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_AUTH_FAILED)) { showSnackbar(t('sources.errors.authFailedSource'), 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_NOT_FOUND)) { showSnackbar(t('sources.errors.sourceDeleted'), 'error'); loadSources(); // Refresh the sources list } else { showSnackbar(errorInfo.message || t('sources.errors.syncStartFailed'), 'error'); } } finally { setSyncingSource(null); } }; const handleDeepScan = async () => { if (!sourceToSync) return; setDeepScanning(true); handleCloseSyncModal(); try { await sourcesService.triggerDeepScan(sourceToSync.id); showSnackbar(t('sources.messages.deepScanSuccess'), 'success'); setTimeout(loadSources, 1000); } catch (error: any) { console.error('Failed to trigger deep scan:', error); if (error.response?.status === 409) { showSnackbar('Source is already syncing', 'warning'); } else if (error.response?.status === 400 && error.response?.data?.message?.includes('only supported for WebDAV')) { showSnackbar(t('sources.errors.deepScanWebdavOnly'), 'warning'); } else { showSnackbar(t('sources.errors.deepScanFailed'), 'error'); } } finally { setDeepScanning(false); } }; const handleStopSync = async (sourceId: string) => { setStoppingSync(sourceId); try { await sourcesService.stopSync(sourceId); showSnackbar(t('sources.messages.syncStopSuccess'), 'success'); setTimeout(loadSources, 1000); } catch (error: any) { console.error('Failed to stop sync:', error); if (error.response?.status === 409) { showSnackbar(t('sources.errors.notSyncing'), 'warning'); } else { showSnackbar(t('sources.errors.syncStopFailed'), 'error'); } } finally { setStoppingSync(null); } }; const handleValidation = async (sourceId: string) => { setValidating(sourceId); try { const response = await api.post(`/sources/${sourceId}/validate`); if (response.data.success) { showSnackbar(response.data.message || 'Validation check started successfully', 'success'); setTimeout(loadSources, 2000); // Reload after 2 seconds to show updated status } else { showSnackbar(response.data.message || 'Failed to start validation check', 'error'); } } catch (error: any) { console.error('Failed to trigger validation:', error); const message = error.response?.data?.message || 'Failed to start validation check'; showSnackbar(message, 'error'); } finally { setValidating(null); } }; // Helper function to render validation status const renderValidationStatus = (source: Source) => { const validationStatus = source.validation_status; const validationScore = source.validation_score; const lastValidationAt = source.last_validation_at; let statusColor = theme.palette.grey[500]; let StatusIcon = HealthIcon; let statusText = t('sources.validation.unknown'); let tooltipText = t('sources.validation.statusUnknown'); if (validationStatus === 'healthy') { statusColor = theme.palette.success.main; StatusIcon = CheckCircleIcon; statusText = t('sources.validation.healthy'); tooltipText = `t('sources.validation.healthScore', { score: validationScore || t('sources.labels.notAvailable') })`; } else if (validationStatus === 'warning') { statusColor = theme.palette.warning.main; StatusIcon = WarningIcon; statusText = t('sources.validation.warning'); tooltipText = `t('sources.validation.healthScore', { score: validationScore || t('sources.labels.notAvailable') }) - Issues detected`; } else if (validationStatus === 'critical') { statusColor = theme.palette.error.main; StatusIcon = CriticalIcon; statusText = t('sources.validation.critical'); tooltipText = `t('sources.validation.healthScore', { score: validationScore || t('sources.labels.notAvailable') }) - Critical issues`; } else if (validationStatus === 'validating') { statusColor = theme.palette.info.main; StatusIcon = HealthIcon; statusText = t('sources.validation.validating'); tooltipText = t('sources.validation.inProgress'); } if (lastValidationAt) { const lastValidation = new Date(lastValidationAt); tooltipText += `\nLast checked: ${formatDistanceToNow(lastValidation)} ago`; } return ( } label={statusText} size="small" sx={{ bgcolor: alpha(statusColor, 0.1), color: statusColor, borderColor: statusColor, border: '1px solid', '& .MuiChip-icon': { color: statusColor, }, }} /> ); }; // Utility functions for folder management const addFolder = () => { if (newFolder && !formData.watch_folders.includes(newFolder)) { setFormData({ ...formData, watch_folders: [...formData.watch_folders, newFolder] }); setNewFolder(''); } }; const removeFolder = (folderToRemove: string) => { setFormData({ ...formData, watch_folders: formData.watch_folders.filter(folder => folder !== folderToRemove) }); }; // Utility functions for file extension management const addFileExtension = () => { if (newExtension && !formData.file_extensions.includes(newExtension)) { setFormData({ ...formData, file_extensions: [...formData.file_extensions, newExtension] }); setNewExtension(''); } }; const removeFileExtension = (extensionToRemove: string) => { setFormData({ ...formData, file_extensions: formData.file_extensions.filter(ext => ext !== extensionToRemove) }); }; // Crawl estimation function const estimateCrawl = async () => { setEstimatingCrawl(true); try { let response; if (editingSource) { // Use the source-specific endpoint for existing sources response = await api.post(`/sources/${editingSource.id}/estimate`); } else { // Use the general endpoint with provided config for new sources response = await api.post('/sources/estimate', { server_url: formData.server_url, username: formData.username, password: formData.password, watch_folders: formData.watch_folders, file_extensions: formData.file_extensions, auto_sync: formData.auto_sync, sync_interval_minutes: formData.sync_interval_minutes, server_type: formData.server_type, }); } setCrawlEstimate(response.data); showSnackbar(t('sources.messages.estimationSuccess'), 'success'); } catch (error) { console.error('Failed to estimate crawl:', error); showSnackbar(t('sources.errors.estimateFailed'), 'error'); } finally { setEstimatingCrawl(false); } }; const getSourceIcon = (sourceType: string) => { switch (sourceType) { case 'webdav': return ; case 's3': return ; case 'local_folder': return ; default: return ; } }; const getStatusIcon = (source: Source) => { if (source.status === 'syncing') { return ; } else if (source.status === 'error') { return ; } else { return ; } }; const getStatusColor = (status: string) => { switch (status) { case 'syncing': return theme.palette.info.main; case 'error': return theme.palette.error.main; default: return theme.palette.success.main; } }; const formatBytes = (bytes: number) => { if (bytes === 0) return '0 B'; const k = 1024; const sizes = ['B', 'KB', 'MB', 'GB', 'TB']; const i = Math.floor(Math.log(bytes) / Math.log(k)); return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i]; }; const StatCard = ({ icon, label, value, color = 'primary', tooltip }: { icon: React.ReactNode; label: string; value: string | number; color?: 'primary' | 'success' | 'warning' | 'error' | 'info'; tooltip?: string; }) => { const card = ( {icon} {typeof value === 'number' ? value.toLocaleString() : value} {label} ); return tooltip ? ( {card} ) : card; }; const renderSourceCard = (source: Source) => ( {/* Header */} {getSourceIcon(source.source_type)} {source.name} } label={`${source.total_documents} docs`} size="small" sx={{ borderRadius: 2, bgcolor: alpha(theme.palette.info.main, 0.1), color: theme.palette.info.main, border: `1px solid ${alpha(theme.palette.info.main, 0.3)}`, fontSize: '0.75rem', fontWeight: 600, }} /> } label={t('sources.stats.ocrCount', { count: source.total_documents_ocr })} size="small" sx={{ borderRadius: 2, bgcolor: alpha(theme.palette.success.main, 0.1), color: theme.palette.success.main, border: `1px solid ${alpha(theme.palette.success.main, 0.3)}`, fontSize: '0.75rem', fontWeight: 600, }} /> {!source.enabled && ( )} {/* Action Buttons */} {source.status === 'syncing' ? ( handleStopSync(source.id)} disabled={stoppingSync === source.id} sx={{ bgcolor: alpha(theme.palette.warning.main, 0.1), '&:hover': { bgcolor: alpha(theme.palette.warning.main, 0.2) }, color: theme.palette.warning.main, }} > {stoppingSync === source.id ? ( ) : ( )} ) : ( handleOpenSyncModal(source)} disabled={syncingSource === source.id || deepScanning || !source.enabled} sx={{ bgcolor: alpha(theme.palette.primary.main, 0.1), '&:hover': { bgcolor: alpha(theme.palette.primary.main, 0.2) }, }} > {syncingSource === source.id ? ( ) : ( )} )} {/* Validation Status Display */} {renderValidationStatus(source)} handleValidation(source.id)} disabled={validating === source.id || source.status === 'syncing' || !source.enabled} size="small" sx={{ bgcolor: alpha(theme.palette.info.main, 0.1), '&:hover': { bgcolor: alpha(theme.palette.info.main, 0.2) }, color: theme.palette.info.main, }} > {validating === source.id ? ( ) : ( )} handleEditSource(source)} sx={{ bgcolor: alpha(theme.palette.grey[500], 0.1), '&:hover': { bgcolor: alpha(theme.palette.grey[500], 0.2) }, }} > navigate(`/ignored-files?sourceType=${source.source_type}&sourceName=${encodeURIComponent(source.name)}&sourceId=${source.id}`)} sx={{ bgcolor: alpha(theme.palette.warning.main, 0.1), '&:hover': { bgcolor: alpha(theme.palette.warning.main, 0.2) }, color: theme.palette.warning.main, }} > handleDeleteSource(source)} sx={{ bgcolor: alpha(theme.palette.error.main, 0.1), '&:hover': { bgcolor: alpha(theme.palette.error.main, 0.2) }, color: theme.palette.error.main, }} > {/* Stats Grid */} } label="Documents Stored" value={source.total_documents} color="info" tooltip="Total number of documents currently stored from this source" /> } label="OCR Processed" value={source.total_documents_ocr} color="success" tooltip="Number of documents that have been successfully OCR'd" /> } label="Last Sync" value={source.last_sync_at ? formatDistanceToNow(new Date(source.last_sync_at), { addSuffix: true }) : t('sources.stats.never')} color="primary" tooltip="When this source was last synchronized" /> } label="Files Pending" value={source.total_files_pending} color="warning" tooltip="Files discovered but not yet processed during sync" /> } label="Total Size" value={formatBytes(source.total_size_bytes)} color="primary" tooltip="Total size of files successfully downloaded from this source" /> {/* Sync Progress Display */} {/* Error Alert */} {source.last_error && ( {source.last_error} {source.last_error_at && ( {formatDistanceToNow(new Date(source.last_error_at), { addSuffix: true })} )} )} ); return ( {/* Header */} Document Sources Connect and manage your document sources with intelligent syncing {/* OCR Controls for Admin Users */} {user?.role === 'Admin' && ( <> {ocrLoading ? ( ) : ocrStatus.is_paused ? ( ) : ( )} )} {/* Content */} {loading ? ( ) : sources.length === 0 ? ( No Sources Configured Connect your first document source to start automatically syncing and processing your files with AI-powered OCR. ) : ( {sources.map(renderSourceCard)} )} {/* Create/Edit Dialog - Enhanced */} setDialogOpen(false)} maxWidth="md" fullWidth PaperProps={{ sx: { borderRadius: 4, background: theme.palette.background.paper, } }} > {editingSource ? : } {editingSource ? t('sources.actions.editSource') : t('sources.dialog.createTitle')} {editingSource ? t('sources.dialog.editSubtitle') : t('sources.dialog.createSubtitle')} setFormData({ ...formData, name: e.target.value })} placeholder="My Document Server" sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2, } }} /> {!editingSource && ( Source Type )} {formData.source_type === 'webdav' && ( WebDAV Configuration setFormData({ ...formData, server_url: e.target.value })} placeholder={ formData.server_type === 'nextcloud' ? "https://nextcloud.example.com/" : formData.server_type === 'owncloud' ? "https://owncloud.example.com/remote.php/dav/files/username/" : "https://webdav.example.com/dav/" } sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> setFormData({ ...formData, username: e.target.value })} sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> setFormData({ ...formData, password: e.target.value })} sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> Server Type setFormData({ ...formData, auto_sync: e.target.checked })} /> } label={ Enable Automatic Sync Automatically sync files on a schedule } /> {formData.auto_sync && ( setFormData({ ...formData, sync_interval_minutes: parseInt(e.target.value) || 60 })} inputProps={{ min: 15, max: 1440 }} helperText="How often to check for new files (15 min - 24 hours)" sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> )} {/* Folder Management */} Folders to Monitor Specify which folders to scan for files. Use absolute paths starting with "/". setNewFolder(e.target.value)} placeholder="/Documents" sx={{ flexGrow: 1, '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> {formData.watch_folders.map((folder, index) => ( removeFolder(folder)} sx={{ mr: 1, mb: 1, borderRadius: 2, bgcolor: alpha(theme.palette.secondary.main, 0.1), color: theme.palette.secondary.main, }} /> ))} {/* File Extensions */} File Extensions File types to sync and process with OCR. setNewExtension(e.target.value)} placeholder="docx" sx={{ flexGrow: 1, '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> {formData.file_extensions.map((extension, index) => ( removeFileExtension(extension)} sx={{ mr: 1, mb: 1, borderRadius: 2, bgcolor: alpha(theme.palette.warning.main, 0.1), color: theme.palette.warning.main, }} /> ))} {/* Crawl Estimation */} {editingSource && formData.server_url && formData.username && formData.watch_folders.length > 0 && ( <> Crawl Estimation Estimate how many files will be processed and how long it will take. {estimatingCrawl && ( Analyzing folders and counting files... )} {crawlEstimate && ( Estimation Results {crawlEstimate.total_files?.toLocaleString() || '0'} Total Files {crawlEstimate.total_supported_files?.toLocaleString() || '0'} Supported Files {crawlEstimate.total_estimated_time_hours?.toFixed(1) || '0'}h Estimated Time {crawlEstimate.total_size_mb ? (crawlEstimate.total_size_mb / 1024).toFixed(1) : '0'}GB Total Size {crawlEstimate.folders && crawlEstimate.folders.length > 0 && ( Folder Total Files Supported Est. Time Size (MB) {crawlEstimate.folders.map((folder: any) => ( {folder.path} {folder.total_files?.toLocaleString() || '0'} {folder.supported_files?.toLocaleString() || '0'} {folder.estimated_time_hours?.toFixed(1) || '0'}h {folder.total_size_mb?.toFixed(1) || '0'} ))}
)}
)} )}
)} {formData.source_type === 'local_folder' && ( Local Folder Configuration Monitor local filesystem directories for new documents. Ensure the application has read access to the specified paths. setFormData({ ...formData, recursive: e.target.checked })} /> } label={ Recursive Scanning Scan subdirectories recursively } /> setFormData({ ...formData, follow_symlinks: e.target.checked })} /> } label={ Follow Symbolic Links Follow symlinks when scanning directories } /> setFormData({ ...formData, auto_sync: e.target.checked })} /> } label={ Enable Automatic Sync Automatically scan for new files on a schedule } /> {formData.auto_sync && ( setFormData({ ...formData, sync_interval_minutes: parseInt(e.target.value) || 60 })} inputProps={{ min: 15, max: 1440 }} helperText="How often to scan for new files (15 min - 24 hours)" sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> )} {/* Folder Management */} Directories to Monitor Specify which local directories to scan for files. Use absolute paths. setNewFolder(e.target.value)} placeholder="/home/user/Documents" sx={{ flexGrow: 1, '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> {formData.watch_folders.map((folder, index) => ( removeFolder(folder)} sx={{ mr: 1, mb: 1, borderRadius: 2, bgcolor: alpha(theme.palette.secondary.main, 0.1), color: theme.palette.secondary.main, }} /> ))} {/* File Extensions */} File Extensions File types to monitor and process with OCR. setNewExtension(e.target.value)} placeholder="docx" sx={{ flexGrow: 1, '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> {formData.file_extensions.map((extension, index) => ( removeFileExtension(extension)} sx={{ mr: 1, mb: 1, borderRadius: 2, bgcolor: alpha(theme.palette.warning.main, 0.1), color: theme.palette.warning.main, }} /> ))} )} {formData.source_type === 's3' && ( S3 Compatible Storage Configuration Connect to AWS S3, MinIO, or any S3-compatible storage service. For MinIO, provide the endpoint URL of your server. setFormData({ ...formData, bucket_name: e.target.value })} placeholder="my-documents-bucket" sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> setFormData({ ...formData, region: e.target.value })} placeholder="us-east-1" sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> setFormData({ ...formData, access_key_id: e.target.value })} sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> setFormData({ ...formData, secret_access_key: e.target.value })} sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> setFormData({ ...formData, endpoint_url: e.target.value })} placeholder="https://minio.example.com (for MinIO/S3-compatible services)" helperText="Leave empty for AWS S3, or provide custom endpoint for MinIO/other S3-compatible storage" sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> setFormData({ ...formData, prefix: e.target.value })} placeholder="documents/" helperText="Optional prefix to limit scanning to specific object keys" sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> setFormData({ ...formData, auto_sync: e.target.checked })} /> } label={ Enable Automatic Sync Automatically check for new objects on a schedule } /> {formData.auto_sync && ( setFormData({ ...formData, sync_interval_minutes: parseInt(e.target.value) || 60 })} inputProps={{ min: 15, max: 1440 }} helperText="How often to check for new objects (15 min - 24 hours)" sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> )} {/* Folder Management (prefixes for S3) */} Object Prefixes to Monitor Specify which object prefixes (like folders) to scan for files. setNewFolder(e.target.value)} placeholder="documents/" sx={{ flexGrow: 1, '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> {formData.watch_folders.map((folder, index) => ( removeFolder(folder)} sx={{ mr: 1, mb: 1, borderRadius: 2, bgcolor: alpha(theme.palette.secondary.main, 0.1), color: theme.palette.secondary.main, }} /> ))} {/* File Extensions */} File Extensions File types to sync and process with OCR. setNewExtension(e.target.value)} placeholder="docx" sx={{ flexGrow: 1, '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> {formData.file_extensions.map((extension, index) => ( removeFileExtension(extension)} sx={{ mr: 1, mb: 1, borderRadius: 2, bgcolor: alpha(theme.palette.warning.main, 0.1), color: theme.palette.warning.main, }} /> ))} )} setFormData({ ...formData, enabled: e.target.checked })} /> } label={ Source Enabled Enable this source for syncing } />
{(formData.source_type === 'webdav' || formData.source_type === 'local_folder' || formData.source_type === 's3') && ( )}
{/* Delete Confirmation Dialog */} Delete Source Are you sure you want to delete "{sourceToDelete?.name}"? This action cannot be undone. The source configuration and all associated sync history will be permanently removed. {/* Sync Type Selection Modal */} Choose Sync Type {sourceToSync && ( <> Select the type of synchronization for {sourceToSync.name}: )} Quick Sync Fast incremental sync using ETags. Only processes new or changed files. Deep Scan Complete rescan that resets ETag expectations. Use for troubleshooting sync issues. {sourceToSync?.source_type === 'webdav' ? ( ) : ( )} {sourceToSync?.source_type !== 'webdav' && ( Deep scan is currently only available for WebDAV sources. Other source types will use quick sync. )} {/* Snackbar */} setSnackbar({ ...snackbar, open: false })} anchorOrigin={{ vertical: 'bottom', horizontal: 'right' }} > setSnackbar({ ...snackbar, open: false })} severity={snackbar.severity} sx={{ width: '100%', borderRadius: 3, }} > {snackbar.message} {/* Custom CSS for animations */}
); }; export default SourcesPage;