import React, { useState, useEffect } from 'react'; import { Box, Container, Typography, Paper, Button, Card, CardContent, Chip, IconButton, Dialog, DialogTitle, DialogContent, DialogContentText, DialogActions, TextField, FormControl, InputLabel, Select, MenuItem, Alert, LinearProgress, Snackbar, Divider, FormControlLabel, Switch, Tooltip, CircularProgress, Fade, Stack, Avatar, Badge, useTheme, alpha, Table, TableBody, TableCell, TableContainer, TableHead, TableRow, } from '@mui/material'; import Grid from '@mui/material/GridLegacy'; import { Add as AddIcon, CloudSync as CloudSyncIcon, Error as ErrorIcon, CheckCircle as CheckCircleIcon, Edit as EditIcon, Delete as DeleteIcon, PlayArrow as PlayArrowIcon, Stop as StopIcon, Storage as StorageIcon, Cloud as CloudIcon, Speed as SpeedIcon, Timeline as TimelineIcon, TrendingUp as TrendingUpIcon, Security as SecurityIcon, AutoFixHigh as AutoFixHighIcon, Sync as SyncIcon, MoreVert as MoreVertIcon, Menu as MenuIcon, Speed as QuickSyncIcon, ManageSearch as DeepScanIcon, Folder as FolderIcon, Assessment as AssessmentIcon, Extension as ExtensionIcon, Storage as ServerIcon, Pause as PauseIcon, PlayArrow as ResumeIcon, TextSnippet as DocumentIcon, Visibility as OcrIcon, Block as BlockIcon, HealthAndSafety as HealthIcon, Warning as WarningIcon, Error as CriticalIcon, } from '@mui/icons-material'; import { useNavigate } from 'react-router-dom'; import api, { queueService, sourcesService, ErrorHelper, ErrorCodes } from '../services/api'; import { formatDistanceToNow } from 'date-fns'; import { useAuth } from '../contexts/AuthContext'; import SyncProgressDisplay from '../components/SyncProgress'; interface Source { id: string; name: string; source_type: 'webdav' | 'local_folder' | 's3'; enabled: boolean; config: any; status: 'idle' | 'syncing' | 'error'; last_sync_at: string | null; last_error: string | null; last_error_at: string | null; total_files_synced: number; total_files_pending: number; total_size_bytes: number; total_documents: number; total_documents_ocr: number; created_at: string; updated_at: string; // Validation fields validation_status?: string | null; last_validation_at?: string | null; validation_score?: number | null; validation_issues?: string | null; } interface SnackbarState { open: boolean; message: string; severity: 'success' | 'error' | 'warning' | 'info'; } const SourcesPage: React.FC = () => { const theme = useTheme(); const navigate = useNavigate(); const { user } = useAuth(); const [sources, setSources] = useState([]); const [loading, setLoading] = useState(true); const [ocrStatus, setOcrStatus] = useState<{ is_paused: boolean; status: string }>({ is_paused: false, status: 'running' }); const [ocrLoading, setOcrLoading] = useState(false); const [dialogOpen, setDialogOpen] = useState(false); const [editingSource, setEditingSource] = useState(null); const [deleteDialogOpen, setDeleteDialogOpen] = useState(false); const [sourceToDelete, setSourceToDelete] = useState(null); const [deleteLoading, setDeleteLoading] = useState(false); const [snackbar, setSnackbar] = useState({ open: false, message: '', severity: 'info', }); // Form state const [formData, setFormData] = useState({ name: '', source_type: 'webdav' as 'webdav' | 'local_folder' | 's3', enabled: true, // WebDAV fields server_url: '', username: '', password: '', server_type: 'generic' as 'nextcloud' | 'owncloud' | 'generic', // Local Folder fields recursive: true, follow_symlinks: false, // S3 fields bucket_name: '', region: 'us-east-1', access_key_id: '', secret_access_key: '', endpoint_url: '', prefix: '', // Common fields watch_folders: ['/Documents'], file_extensions: ['pdf', 'png', 'jpg', 'jpeg', 'tiff', 'bmp', 'txt'], auto_sync: false, sync_interval_minutes: 60, }); // Additional state for enhanced features const [newFolder, setNewFolder] = useState(''); const [newExtension, setNewExtension] = useState(''); const [crawlEstimate, setCrawlEstimate] = useState(null); const [estimatingCrawl, setEstimatingCrawl] = useState(false); const [testingConnection, setTestingConnection] = useState(false); const [syncingSource, setSyncingSource] = useState(null); const [stoppingSync, setStoppingSync] = useState(null); const [validating, setValidating] = useState(null); const [autoRefreshing, setAutoRefreshing] = useState(false); // Sync modal state const [syncModalOpen, setSyncModalOpen] = useState(false); const [sourceToSync, setSourceToSync] = useState(null); const [deepScanning, setDeepScanning] = useState(false); useEffect(() => { loadSources(); if (user?.role === 'Admin') { loadOcrStatus(); } }, [user]); // Auto-refresh sources when any source is syncing useEffect(() => { const activeSyncingSources = sources.filter(source => source.status === 'syncing'); if (activeSyncingSources.length > 0) { setAutoRefreshing(true); const interval = setInterval(() => { loadSources(); }, 5000); // Poll every 5 seconds during active sync return () => { clearInterval(interval); setAutoRefreshing(false); }; } else { setAutoRefreshing(false); } }, [sources]); // Update default folders when source type changes useEffect(() => { if (!editingSource) { // Only for new sources let defaultFolders; switch (formData.source_type) { case 'local_folder': defaultFolders = ['/home/user/Documents']; break; case 's3': defaultFolders = ['documents/']; break; case 'webdav': default: defaultFolders = ['/Documents']; break; } setFormData(prev => ({ ...prev, watch_folders: defaultFolders })); } }, [formData.source_type, editingSource]); const loadSources = async () => { try { const response = await api.get('/sources'); setSources(response.data); } catch (error) { console.error('Failed to load sources:', error); showSnackbar('Failed to load sources', 'error'); } finally { setLoading(false); } }; const showSnackbar = (message: string, severity: SnackbarState['severity']) => { setSnackbar({ open: true, message, severity }); }; // OCR Control Functions (Admin only) const loadOcrStatus = async () => { if (user?.role !== 'Admin') return; try { const response = await queueService.getOcrStatus(); setOcrStatus(response.data); } catch (error) { console.error('Failed to load OCR status:', error); } }; const handlePauseOcr = async () => { if (user?.role !== 'Admin') return; setOcrLoading(true); try { await queueService.pauseOcr(); await loadOcrStatus(); showSnackbar('OCR processing paused successfully', 'success'); } catch (error) { console.error('Failed to pause OCR:', error); showSnackbar('Failed to pause OCR processing', 'error'); } finally { setOcrLoading(false); } }; const handleResumeOcr = async () => { if (user?.role !== 'Admin') return; setOcrLoading(true); try { await queueService.resumeOcr(); await loadOcrStatus(); showSnackbar('OCR processing resumed successfully', 'success'); } catch (error) { console.error('Failed to resume OCR:', error); showSnackbar('Failed to resume OCR processing', 'error'); } finally { setOcrLoading(false); } }; const handleCreateSource = () => { setEditingSource(null); setFormData({ name: '', source_type: 'webdav', enabled: true, // WebDAV fields server_url: '', username: '', password: '', server_type: 'generic', // Local Folder fields recursive: true, follow_symlinks: false, // S3 fields bucket_name: '', region: 'us-east-1', access_key_id: '', secret_access_key: '', endpoint_url: '', prefix: '', // Common fields watch_folders: ['/Documents'], file_extensions: ['pdf', 'png', 'jpg', 'jpeg', 'tiff', 'bmp', 'txt'], auto_sync: false, sync_interval_minutes: 60, }); setCrawlEstimate(null); setNewFolder(''); setNewExtension(''); setDialogOpen(true); }; const handleEditSource = (source: Source) => { setEditingSource(source); const config = source.config; setFormData({ name: source.name, source_type: source.source_type, enabled: source.enabled, // WebDAV fields server_url: config.server_url || '', username: config.username || '', password: config.password || '', server_type: config.server_type || 'generic', // Local Folder fields recursive: config.recursive !== undefined ? config.recursive : true, follow_symlinks: config.follow_symlinks || false, // S3 fields bucket_name: config.bucket_name || '', region: config.region || 'us-east-1', access_key_id: config.access_key_id || '', secret_access_key: config.secret_access_key || '', endpoint_url: config.endpoint_url || '', prefix: config.prefix || '', // Common fields watch_folders: config.watch_folders || ['/Documents'], file_extensions: config.file_extensions || ['pdf', 'png', 'jpg', 'jpeg', 'tiff', 'bmp', 'txt'], auto_sync: config.auto_sync || false, sync_interval_minutes: config.sync_interval_minutes || 60, }); setCrawlEstimate(null); setNewFolder(''); setNewExtension(''); setDialogOpen(true); }; const handleSaveSource = async () => { try { let config = {}; // Build config based on source type if (formData.source_type === 'webdav') { config = { server_url: formData.server_url, username: formData.username, password: formData.password, watch_folders: formData.watch_folders, file_extensions: formData.file_extensions, auto_sync: formData.auto_sync, sync_interval_minutes: formData.sync_interval_minutes, server_type: formData.server_type, }; } else if (formData.source_type === 'local_folder') { config = { watch_folders: formData.watch_folders, file_extensions: formData.file_extensions, auto_sync: formData.auto_sync, sync_interval_minutes: formData.sync_interval_minutes, recursive: formData.recursive, follow_symlinks: formData.follow_symlinks, }; } else if (formData.source_type === 's3') { config = { bucket_name: formData.bucket_name, region: formData.region, access_key_id: formData.access_key_id, secret_access_key: formData.secret_access_key, endpoint_url: formData.endpoint_url, prefix: formData.prefix, watch_folders: formData.watch_folders, file_extensions: formData.file_extensions, auto_sync: formData.auto_sync, sync_interval_minutes: formData.sync_interval_minutes, }; } if (editingSource) { await api.put(`/sources/${editingSource.id}`, { name: formData.name, enabled: formData.enabled, config, }); showSnackbar('Source updated successfully', 'success'); } else { await api.post('/sources', { name: formData.name, source_type: formData.source_type, enabled: formData.enabled, config, }); showSnackbar('Source created successfully', 'success'); } setDialogOpen(false); loadSources(); } catch (error) { console.error('Failed to save source:', error); const errorInfo = ErrorHelper.formatErrorForDisplay(error, true); // Handle specific source errors if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_DUPLICATE_NAME)) { showSnackbar('A source with this name already exists. Please choose a different name.', 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_CONFIG_INVALID)) { showSnackbar('Source configuration is invalid. Please check your settings and try again.', 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_AUTH_FAILED)) { showSnackbar('Authentication failed. Please verify your credentials.', 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_CONNECTION_FAILED)) { showSnackbar('Cannot connect to the source. Please check your network and server settings.', 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_INVALID_PATH)) { showSnackbar('Invalid path specified. Please check your folder paths and try again.', 'error'); } else { showSnackbar(errorInfo.message || 'Failed to save source', 'error'); } } }; const handleDeleteSource = (source: Source) => { setSourceToDelete(source); setDeleteDialogOpen(true); }; const handleDeleteCancel = () => { setDeleteDialogOpen(false); setSourceToDelete(null); setDeleteLoading(false); }; const handleDeleteConfirm = async () => { if (!sourceToDelete) return; setDeleteLoading(true); try { await api.delete(`/sources/${sourceToDelete.id}`); showSnackbar('Source deleted successfully', 'success'); loadSources(); handleDeleteCancel(); } catch (error) { console.error('Failed to delete source:', error); const errorInfo = ErrorHelper.formatErrorForDisplay(error, true); // Handle specific delete errors if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_NOT_FOUND)) { showSnackbar('Source not found. It may have already been deleted.', 'warning'); loadSources(); // Refresh the list handleDeleteCancel(); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_SYNC_IN_PROGRESS)) { showSnackbar('Cannot delete source while sync is in progress. Please stop the sync first.', 'error'); } else { showSnackbar(errorInfo.message || 'Failed to delete source', 'error'); } setDeleteLoading(false); } }; const handleTestConnection = async () => { setTestingConnection(true); try { let response; if (formData.source_type === 'webdav') { response = await api.post('/sources/test-connection', { source_type: 'webdav', config: { server_url: formData.server_url, username: formData.username, password: formData.password, server_type: formData.server_type, watch_folders: formData.watch_folders, file_extensions: formData.file_extensions, } }); } else if (formData.source_type === 'local_folder') { response = await api.post('/sources/test-connection', { source_type: 'local_folder', config: { watch_folders: formData.watch_folders, file_extensions: formData.file_extensions, recursive: formData.recursive, follow_symlinks: formData.follow_symlinks, } }); } else if (formData.source_type === 's3') { response = await api.post('/sources/test-connection', { source_type: 's3', config: { bucket_name: formData.bucket_name, region: formData.region, access_key_id: formData.access_key_id, secret_access_key: formData.secret_access_key, endpoint_url: formData.endpoint_url, prefix: formData.prefix, } }); } if (response && response.data.success) { showSnackbar(response.data.message || 'Connection successful!', 'success'); } else { showSnackbar(response?.data.message || 'Connection failed', 'error'); } } catch (error: any) { console.error('Failed to test connection:', error); const errorInfo = ErrorHelper.formatErrorForDisplay(error, true); // Handle specific connection test errors if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_CONNECTION_FAILED)) { showSnackbar('Connection failed. Please check your server URL and network connectivity.', 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_AUTH_FAILED)) { showSnackbar('Authentication failed. Please verify your username and password.', 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_INVALID_PATH)) { showSnackbar('Invalid path specified. Please check your folder paths.', 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_CONFIG_INVALID)) { showSnackbar('Configuration is invalid. Please review your settings.', 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_NETWORK_TIMEOUT)) { showSnackbar('Connection timed out. Please check your network and try again.', 'error'); } else { showSnackbar(errorInfo.message || 'Failed to test connection', 'error'); } } finally { setTestingConnection(false); } }; // Open sync modal instead of directly triggering sync const handleOpenSyncModal = (source: Source) => { setSourceToSync(source); setSyncModalOpen(true); }; const handleCloseSyncModal = () => { setSyncModalOpen(false); setSourceToSync(null); }; const handleQuickSync = async () => { if (!sourceToSync) return; setSyncingSource(sourceToSync.id); handleCloseSyncModal(); try { await sourcesService.triggerSync(sourceToSync.id); showSnackbar('Quick sync started successfully', 'success'); setTimeout(loadSources, 1000); } catch (error: any) { console.error('Failed to trigger sync:', error); const errorInfo = ErrorHelper.formatErrorForDisplay(error, true); // Handle specific sync errors if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_SYNC_IN_PROGRESS)) { showSnackbar('Source is already syncing. Please wait for the current sync to complete.', 'warning'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_CONNECTION_FAILED)) { showSnackbar('Cannot connect to source. Please check your connection and try again.', 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_AUTH_FAILED)) { showSnackbar('Authentication failed. Please verify your source credentials.', 'error'); } else if (ErrorHelper.isErrorCode(error, ErrorCodes.SOURCE_NOT_FOUND)) { showSnackbar('Source not found. It may have been deleted.', 'error'); loadSources(); // Refresh the sources list } else { showSnackbar(errorInfo.message || 'Failed to start sync', 'error'); } } finally { setSyncingSource(null); } }; const handleDeepScan = async () => { if (!sourceToSync) return; setDeepScanning(true); handleCloseSyncModal(); try { await sourcesService.triggerDeepScan(sourceToSync.id); showSnackbar('Deep scan started successfully', 'success'); setTimeout(loadSources, 1000); } catch (error: any) { console.error('Failed to trigger deep scan:', error); if (error.response?.status === 409) { showSnackbar('Source is already syncing', 'warning'); } else if (error.response?.status === 400 && error.response?.data?.message?.includes('only supported for WebDAV')) { showSnackbar('Deep scan is only supported for WebDAV sources', 'warning'); } else { showSnackbar('Failed to start deep scan', 'error'); } } finally { setDeepScanning(false); } }; const handleStopSync = async (sourceId: string) => { setStoppingSync(sourceId); try { await sourcesService.stopSync(sourceId); showSnackbar('Sync stopped successfully', 'success'); setTimeout(loadSources, 1000); } catch (error: any) { console.error('Failed to stop sync:', error); if (error.response?.status === 409) { showSnackbar('Source is not currently syncing', 'warning'); } else { showSnackbar('Failed to stop sync', 'error'); } } finally { setStoppingSync(null); } }; const handleValidation = async (sourceId: string) => { setValidating(sourceId); try { const response = await api.post(`/sources/${sourceId}/validate`); if (response.data.success) { showSnackbar(response.data.message || 'Validation check started successfully', 'success'); setTimeout(loadSources, 2000); // Reload after 2 seconds to show updated status } else { showSnackbar(response.data.message || 'Failed to start validation check', 'error'); } } catch (error: any) { console.error('Failed to trigger validation:', error); const message = error.response?.data?.message || 'Failed to start validation check'; showSnackbar(message, 'error'); } finally { setValidating(null); } }; // Helper function to render validation status const renderValidationStatus = (source: Source) => { const validationStatus = source.validation_status; const validationScore = source.validation_score; const lastValidationAt = source.last_validation_at; let statusColor = theme.palette.grey[500]; let StatusIcon = HealthIcon; let statusText = 'Unknown'; let tooltipText = 'Validation status unknown'; if (validationStatus === 'healthy') { statusColor = theme.palette.success.main; StatusIcon = CheckCircleIcon; statusText = 'Healthy'; tooltipText = `Health score: ${validationScore || 'N/A'}`; } else if (validationStatus === 'warning') { statusColor = theme.palette.warning.main; StatusIcon = WarningIcon; statusText = 'Warning'; tooltipText = `Health score: ${validationScore || 'N/A'} - Issues detected`; } else if (validationStatus === 'critical') { statusColor = theme.palette.error.main; StatusIcon = CriticalIcon; statusText = 'Critical'; tooltipText = `Health score: ${validationScore || 'N/A'} - Critical issues`; } else if (validationStatus === 'validating') { statusColor = theme.palette.info.main; StatusIcon = HealthIcon; statusText = 'Validating'; tooltipText = 'Validation check in progress'; } if (lastValidationAt) { const lastValidation = new Date(lastValidationAt); tooltipText += `\nLast checked: ${formatDistanceToNow(lastValidation)} ago`; } return ( } label={statusText} size="small" sx={{ bgcolor: alpha(statusColor, 0.1), color: statusColor, borderColor: statusColor, border: '1px solid', '& .MuiChip-icon': { color: statusColor, }, }} /> ); }; // Utility functions for folder management const addFolder = () => { if (newFolder && !formData.watch_folders.includes(newFolder)) { setFormData({ ...formData, watch_folders: [...formData.watch_folders, newFolder] }); setNewFolder(''); } }; const removeFolder = (folderToRemove: string) => { setFormData({ ...formData, watch_folders: formData.watch_folders.filter(folder => folder !== folderToRemove) }); }; // Utility functions for file extension management const addFileExtension = () => { if (newExtension && !formData.file_extensions.includes(newExtension)) { setFormData({ ...formData, file_extensions: [...formData.file_extensions, newExtension] }); setNewExtension(''); } }; const removeFileExtension = (extensionToRemove: string) => { setFormData({ ...formData, file_extensions: formData.file_extensions.filter(ext => ext !== extensionToRemove) }); }; // Crawl estimation function const estimateCrawl = async () => { setEstimatingCrawl(true); try { let response; if (editingSource) { // Use the source-specific endpoint for existing sources response = await api.post(`/sources/${editingSource.id}/estimate`); } else { // Use the general endpoint with provided config for new sources response = await api.post('/sources/estimate', { server_url: formData.server_url, username: formData.username, password: formData.password, watch_folders: formData.watch_folders, file_extensions: formData.file_extensions, auto_sync: formData.auto_sync, sync_interval_minutes: formData.sync_interval_minutes, server_type: formData.server_type, }); } setCrawlEstimate(response.data); showSnackbar('Crawl estimation completed', 'success'); } catch (error) { console.error('Failed to estimate crawl:', error); showSnackbar('Failed to estimate crawl', 'error'); } finally { setEstimatingCrawl(false); } }; const getSourceIcon = (sourceType: string) => { switch (sourceType) { case 'webdav': return ; case 's3': return ; case 'local_folder': return ; default: return ; } }; const getStatusIcon = (source: Source) => { if (source.status === 'syncing') { return ; } else if (source.status === 'error') { return ; } else { return ; } }; const getStatusColor = (status: string) => { switch (status) { case 'syncing': return theme.palette.info.main; case 'error': return theme.palette.error.main; default: return theme.palette.success.main; } }; const formatBytes = (bytes: number) => { if (bytes === 0) return '0 B'; const k = 1024; const sizes = ['B', 'KB', 'MB', 'GB', 'TB']; const i = Math.floor(Math.log(bytes) / Math.log(k)); return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i]; }; const StatCard = ({ icon, label, value, color = 'primary', tooltip }: { icon: React.ReactNode; label: string; value: string | number; color?: 'primary' | 'success' | 'warning' | 'error' | 'info'; tooltip?: string; }) => { const card = ( {icon} {typeof value === 'number' ? value.toLocaleString() : value} {label} ); return tooltip ? ( {card} ) : card; }; const renderSourceCard = (source: Source) => ( {/* Header */} {getSourceIcon(source.source_type)} {source.name} } label={`${source.total_documents} docs`} size="small" sx={{ borderRadius: 2, bgcolor: alpha(theme.palette.info.main, 0.1), color: theme.palette.info.main, border: `1px solid ${alpha(theme.palette.info.main, 0.3)}`, fontSize: '0.75rem', fontWeight: 600, }} /> } label={`${source.total_documents_ocr} OCR'd`} size="small" sx={{ borderRadius: 2, bgcolor: alpha(theme.palette.success.main, 0.1), color: theme.palette.success.main, border: `1px solid ${alpha(theme.palette.success.main, 0.3)}`, fontSize: '0.75rem', fontWeight: 600, }} /> {!source.enabled && ( )} {/* Action Buttons */} {source.status === 'syncing' ? ( handleStopSync(source.id)} disabled={stoppingSync === source.id} sx={{ bgcolor: alpha(theme.palette.warning.main, 0.1), '&:hover': { bgcolor: alpha(theme.palette.warning.main, 0.2) }, color: theme.palette.warning.main, }} > {stoppingSync === source.id ? ( ) : ( )} ) : ( handleOpenSyncModal(source)} disabled={syncingSource === source.id || deepScanning || !source.enabled} sx={{ bgcolor: alpha(theme.palette.primary.main, 0.1), '&:hover': { bgcolor: alpha(theme.palette.primary.main, 0.2) }, }} > {syncingSource === source.id ? ( ) : ( )} )} {/* Validation Status Display */} {renderValidationStatus(source)} handleValidation(source.id)} disabled={validating === source.id || source.status === 'syncing' || !source.enabled} size="small" sx={{ bgcolor: alpha(theme.palette.info.main, 0.1), '&:hover': { bgcolor: alpha(theme.palette.info.main, 0.2) }, color: theme.palette.info.main, }} > {validating === source.id ? ( ) : ( )} handleEditSource(source)} sx={{ bgcolor: alpha(theme.palette.grey[500], 0.1), '&:hover': { bgcolor: alpha(theme.palette.grey[500], 0.2) }, }} > navigate(`/ignored-files?sourceType=${source.source_type}&sourceName=${encodeURIComponent(source.name)}&sourceId=${source.id}`)} sx={{ bgcolor: alpha(theme.palette.warning.main, 0.1), '&:hover': { bgcolor: alpha(theme.palette.warning.main, 0.2) }, color: theme.palette.warning.main, }} > handleDeleteSource(source)} sx={{ bgcolor: alpha(theme.palette.error.main, 0.1), '&:hover': { bgcolor: alpha(theme.palette.error.main, 0.2) }, color: theme.palette.error.main, }} > {/* Stats Grid */} } label="Documents Stored" value={source.total_documents} color="info" tooltip="Total number of documents currently stored from this source" /> } label="OCR Processed" value={source.total_documents_ocr} color="success" tooltip="Number of documents that have been successfully OCR'd" /> } label="Last Sync" value={source.last_sync_at ? formatDistanceToNow(new Date(source.last_sync_at), { addSuffix: true }) : 'Never'} color="primary" tooltip="When this source was last synchronized" /> } label="Files Pending" value={source.total_files_pending} color="warning" tooltip="Files discovered but not yet processed during sync" /> } label="Total Size" value={formatBytes(source.total_size_bytes)} color="primary" tooltip="Total size of files successfully downloaded from this source" /> {/* Sync Progress Display */} {/* Error Alert */} {source.last_error && ( {source.last_error} {source.last_error_at && ( {formatDistanceToNow(new Date(source.last_error_at), { addSuffix: true })} )} )} ); return ( {/* Header */} Document Sources Connect and manage your document sources with intelligent syncing {/* OCR Controls for Admin Users */} {user?.role === 'Admin' && ( <> {ocrLoading ? ( ) : ocrStatus.is_paused ? ( ) : ( )} )} {/* Content */} {loading ? ( ) : sources.length === 0 ? ( No Sources Configured Connect your first document source to start automatically syncing and processing your files with AI-powered OCR. ) : ( {sources.map(renderSourceCard)} )} {/* Create/Edit Dialog - Enhanced */} setDialogOpen(false)} maxWidth="md" fullWidth PaperProps={{ sx: { borderRadius: 4, background: theme.palette.background.paper, } }} > {editingSource ? : } {editingSource ? 'Edit Source' : 'Create New Source'} {editingSource ? 'Update your source configuration' : 'Connect a new document source'} setFormData({ ...formData, name: e.target.value })} placeholder="My Document Server" sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2, } }} /> {!editingSource && ( Source Type )} {formData.source_type === 'webdav' && ( WebDAV Configuration setFormData({ ...formData, server_url: e.target.value })} placeholder={ formData.server_type === 'nextcloud' ? "https://nextcloud.example.com/" : formData.server_type === 'owncloud' ? "https://owncloud.example.com/remote.php/dav/files/username/" : "https://webdav.example.com/dav/" } sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> setFormData({ ...formData, username: e.target.value })} sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> setFormData({ ...formData, password: e.target.value })} sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> Server Type setFormData({ ...formData, auto_sync: e.target.checked })} /> } label={ Enable Automatic Sync Automatically sync files on a schedule } /> {formData.auto_sync && ( setFormData({ ...formData, sync_interval_minutes: parseInt(e.target.value) || 60 })} inputProps={{ min: 15, max: 1440 }} helperText="How often to check for new files (15 min - 24 hours)" sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> )} {/* Folder Management */} Folders to Monitor Specify which folders to scan for files. Use absolute paths starting with "/". setNewFolder(e.target.value)} placeholder="/Documents" sx={{ flexGrow: 1, '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> {formData.watch_folders.map((folder, index) => ( removeFolder(folder)} sx={{ mr: 1, mb: 1, borderRadius: 2, bgcolor: alpha(theme.palette.secondary.main, 0.1), color: theme.palette.secondary.main, }} /> ))} {/* File Extensions */} File Extensions File types to sync and process with OCR. setNewExtension(e.target.value)} placeholder="docx" sx={{ flexGrow: 1, '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> {formData.file_extensions.map((extension, index) => ( removeFileExtension(extension)} sx={{ mr: 1, mb: 1, borderRadius: 2, bgcolor: alpha(theme.palette.warning.main, 0.1), color: theme.palette.warning.main, }} /> ))} {/* Crawl Estimation */} {editingSource && formData.server_url && formData.username && formData.watch_folders.length > 0 && ( <> Crawl Estimation Estimate how many files will be processed and how long it will take. {estimatingCrawl && ( Analyzing folders and counting files... )} {crawlEstimate && ( Estimation Results {crawlEstimate.total_files?.toLocaleString() || '0'} Total Files {crawlEstimate.total_supported_files?.toLocaleString() || '0'} Supported Files {crawlEstimate.total_estimated_time_hours?.toFixed(1) || '0'}h Estimated Time {crawlEstimate.total_size_mb ? (crawlEstimate.total_size_mb / 1024).toFixed(1) : '0'}GB Total Size {crawlEstimate.folders && crawlEstimate.folders.length > 0 && ( Folder Total Files Supported Est. Time Size (MB) {crawlEstimate.folders.map((folder: any) => ( {folder.path} {folder.total_files?.toLocaleString() || '0'} {folder.supported_files?.toLocaleString() || '0'} {folder.estimated_time_hours?.toFixed(1) || '0'}h {folder.total_size_mb?.toFixed(1) || '0'} ))}
)}
)} )}
)} {formData.source_type === 'local_folder' && ( Local Folder Configuration Monitor local filesystem directories for new documents. Ensure the application has read access to the specified paths. setFormData({ ...formData, recursive: e.target.checked })} /> } label={ Recursive Scanning Scan subdirectories recursively } /> setFormData({ ...formData, follow_symlinks: e.target.checked })} /> } label={ Follow Symbolic Links Follow symlinks when scanning directories } /> setFormData({ ...formData, auto_sync: e.target.checked })} /> } label={ Enable Automatic Sync Automatically scan for new files on a schedule } /> {formData.auto_sync && ( setFormData({ ...formData, sync_interval_minutes: parseInt(e.target.value) || 60 })} inputProps={{ min: 15, max: 1440 }} helperText="How often to scan for new files (15 min - 24 hours)" sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> )} {/* Folder Management */} Directories to Monitor Specify which local directories to scan for files. Use absolute paths. setNewFolder(e.target.value)} placeholder="/home/user/Documents" sx={{ flexGrow: 1, '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> {formData.watch_folders.map((folder, index) => ( removeFolder(folder)} sx={{ mr: 1, mb: 1, borderRadius: 2, bgcolor: alpha(theme.palette.secondary.main, 0.1), color: theme.palette.secondary.main, }} /> ))} {/* File Extensions */} File Extensions File types to monitor and process with OCR. setNewExtension(e.target.value)} placeholder="docx" sx={{ flexGrow: 1, '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> {formData.file_extensions.map((extension, index) => ( removeFileExtension(extension)} sx={{ mr: 1, mb: 1, borderRadius: 2, bgcolor: alpha(theme.palette.warning.main, 0.1), color: theme.palette.warning.main, }} /> ))} )} {formData.source_type === 's3' && ( S3 Compatible Storage Configuration Connect to AWS S3, MinIO, or any S3-compatible storage service. For MinIO, provide the endpoint URL of your server. setFormData({ ...formData, bucket_name: e.target.value })} placeholder="my-documents-bucket" sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> setFormData({ ...formData, region: e.target.value })} placeholder="us-east-1" sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> setFormData({ ...formData, access_key_id: e.target.value })} sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> setFormData({ ...formData, secret_access_key: e.target.value })} sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> setFormData({ ...formData, endpoint_url: e.target.value })} placeholder="https://minio.example.com (for MinIO/S3-compatible services)" helperText="Leave empty for AWS S3, or provide custom endpoint for MinIO/other S3-compatible storage" sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> setFormData({ ...formData, prefix: e.target.value })} placeholder="documents/" helperText="Optional prefix to limit scanning to specific object keys" sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> setFormData({ ...formData, auto_sync: e.target.checked })} /> } label={ Enable Automatic Sync Automatically check for new objects on a schedule } /> {formData.auto_sync && ( setFormData({ ...formData, sync_interval_minutes: parseInt(e.target.value) || 60 })} inputProps={{ min: 15, max: 1440 }} helperText="How often to check for new objects (15 min - 24 hours)" sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> )} {/* Folder Management (prefixes for S3) */} Object Prefixes to Monitor Specify which object prefixes (like folders) to scan for files. setNewFolder(e.target.value)} placeholder="documents/" sx={{ flexGrow: 1, '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> {formData.watch_folders.map((folder, index) => ( removeFolder(folder)} sx={{ mr: 1, mb: 1, borderRadius: 2, bgcolor: alpha(theme.palette.secondary.main, 0.1), color: theme.palette.secondary.main, }} /> ))} {/* File Extensions */} File Extensions File types to sync and process with OCR. setNewExtension(e.target.value)} placeholder="docx" sx={{ flexGrow: 1, '& .MuiOutlinedInput-root': { borderRadius: 2 } }} /> {formData.file_extensions.map((extension, index) => ( removeFileExtension(extension)} sx={{ mr: 1, mb: 1, borderRadius: 2, bgcolor: alpha(theme.palette.warning.main, 0.1), color: theme.palette.warning.main, }} /> ))} )} setFormData({ ...formData, enabled: e.target.checked })} /> } label={ Source Enabled Enable this source for syncing } />
{(formData.source_type === 'webdav' || formData.source_type === 'local_folder' || formData.source_type === 's3') && ( )}
{/* Delete Confirmation Dialog */} Delete Source Are you sure you want to delete "{sourceToDelete?.name}"? This action cannot be undone. The source configuration and all associated sync history will be permanently removed. {/* Sync Type Selection Modal */} Choose Sync Type {sourceToSync && ( <> Select the type of synchronization for {sourceToSync.name}: )} Quick Sync Fast incremental sync using ETags. Only processes new or changed files. Deep Scan Complete rescan that resets ETag expectations. Use for troubleshooting sync issues. {sourceToSync?.source_type === 'webdav' ? ( ) : ( )} {sourceToSync?.source_type !== 'webdav' && ( Deep scan is currently only available for WebDAV sources. Other source types will use quick sync. )} {/* Snackbar */} setSnackbar({ ...snackbar, open: false })} anchorOrigin={{ vertical: 'bottom', horizontal: 'right' }} > setSnackbar({ ...snackbar, open: false })} severity={snackbar.severity} sx={{ width: '100%', borderRadius: 3, }} > {snackbar.message} {/* Custom CSS for animations */}
); }; export default SourcesPage;