From 6004f3a00100a6a8ee79441fbfd6ddb12c0d67a8 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Sun, 15 Jun 2025 18:00:35 +0000 Subject: [PATCH] feat(client): also show settings for s3 and local sources in the client --- frontend/src/pages/SourcesPage.tsx | 609 +++++++++++++++++++++++++++-- src/routes/sources.rs | 163 +++++++- 2 files changed, 741 insertions(+), 31 deletions(-) diff --git a/frontend/src/pages/SourcesPage.tsx b/frontend/src/pages/SourcesPage.tsx index 23570c8..9876a7f 100644 --- a/frontend/src/pages/SourcesPage.tsx +++ b/frontend/src/pages/SourcesPage.tsx @@ -143,6 +143,26 @@ const SourcesPage: React.FC = () => { loadSources(); }, []); + // Update default folders when source type changes + useEffect(() => { + if (!editingSource) { // Only for new sources + let defaultFolders; + switch (formData.source_type) { + case 'local_folder': + defaultFolders = ['/home/user/Documents']; + break; + case 's3': + defaultFolders = ['documents/']; + break; + case 'webdav': + default: + defaultFolders = ['/Documents']; + break; + } + setFormData(prev => ({ ...prev, watch_folders: defaultFolders })); + } + }, [formData.source_type, editingSource]); + const loadSources = async () => { try { const response = await api.get('/sources'); @@ -228,16 +248,43 @@ const SourcesPage: React.FC = () => { const handleSaveSource = async () => { try { - const config = { - server_url: formData.server_url, - username: formData.username, - password: formData.password, - watch_folders: formData.watch_folders, - file_extensions: formData.file_extensions, - auto_sync: formData.auto_sync, - sync_interval_minutes: formData.sync_interval_minutes, - server_type: formData.server_type, - }; + let config = {}; + + // Build config based on source type + if (formData.source_type === 'webdav') { + config = { + server_url: formData.server_url, + username: formData.username, + password: formData.password, + watch_folders: formData.watch_folders, + file_extensions: formData.file_extensions, + auto_sync: formData.auto_sync, + sync_interval_minutes: formData.sync_interval_minutes, + server_type: formData.server_type, + }; + } else if (formData.source_type === 'local_folder') { + config = { + watch_folders: formData.watch_folders, + file_extensions: formData.file_extensions, + auto_sync: formData.auto_sync, + sync_interval_minutes: formData.sync_interval_minutes, + recursive: formData.recursive, + follow_symlinks: formData.follow_symlinks, + }; + } else if (formData.source_type === 's3') { + config = { + bucket_name: formData.bucket_name, + region: formData.region, + access_key_id: formData.access_key_id, + secret_access_key: formData.secret_access_key, + endpoint_url: formData.endpoint_url, + prefix: formData.prefix, + watch_folders: formData.watch_folders, + file_extensions: formData.file_extensions, + auto_sync: formData.auto_sync, + sync_interval_minutes: formData.sync_interval_minutes, + }; + } if (editingSource) { await api.put(`/sources/${editingSource.id}`, { @@ -282,20 +329,47 @@ const SourcesPage: React.FC = () => { const handleTestConnection = async () => { setTestingConnection(true); try { - const response = await api.post('/webdav/test-connection', { - server_url: formData.server_url, - username: formData.username, - password: formData.password, - server_type: formData.server_type, - }); - if (response.data.success) { - showSnackbar('Connection successful!', 'success'); - } else { - showSnackbar(response.data.message || 'Connection failed', 'error'); + let response; + if (formData.source_type === 'webdav') { + response = await api.post('/webdav/test-connection', { + server_url: formData.server_url, + username: formData.username, + password: formData.password, + server_type: formData.server_type, + }); + } else if (formData.source_type === 'local_folder') { + response = await api.post('/sources/test-connection', { + source_type: 'local_folder', + config: { + watch_folders: formData.watch_folders, + file_extensions: formData.file_extensions, + recursive: formData.recursive, + follow_symlinks: formData.follow_symlinks, + } + }); + } else if (formData.source_type === 's3') { + response = await api.post('/sources/test-connection', { + source_type: 's3', + config: { + bucket_name: formData.bucket_name, + region: formData.region, + access_key_id: formData.access_key_id, + secret_access_key: formData.secret_access_key, + endpoint_url: formData.endpoint_url, + prefix: formData.prefix, + } + }); } - } catch (error) { + + if (response && response.data.success) { + showSnackbar(response.data.message || 'Connection successful!', 'success'); + } else { + showSnackbar(response?.data.message || 'Connection failed', 'error'); + } + } catch (error: any) { console.error('Failed to test connection:', error); - showSnackbar('Failed to test connection', 'error'); + const errorMessage = error.response?.data?.message || error.message || 'Failed to test connection'; + showSnackbar(errorMessage, 'error'); } finally { setTestingConnection(false); } @@ -391,9 +465,9 @@ const SourcesPage: React.FC = () => { case 'webdav': return ; case 's3': - return ; + return ; case 'local_folder': - return ; + return ; default: return ; } @@ -1282,6 +1356,485 @@ const SourcesPage: React.FC = () => { )} + {formData.source_type === 'local_folder' && ( + + + + + + + + Local Folder Configuration + + + + + + Monitor local filesystem directories for new documents. + Ensure the application has read access to the specified paths. + + + + setFormData({ ...formData, recursive: e.target.checked })} + /> + } + label={ + + + Recursive Scanning + + + Scan subdirectories recursively + + + } + /> + + setFormData({ ...formData, follow_symlinks: e.target.checked })} + /> + } + label={ + + + Follow Symbolic Links + + + Follow symlinks when scanning directories + + + } + /> + + setFormData({ ...formData, auto_sync: e.target.checked })} + /> + } + label={ + + + Enable Automatic Sync + + + Automatically scan for new files on a schedule + + + } + /> + + {formData.auto_sync && ( + setFormData({ ...formData, sync_interval_minutes: parseInt(e.target.value) || 60 })} + inputProps={{ min: 15, max: 1440 }} + helperText="How often to scan for new files (15 min - 24 hours)" + sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} + /> + )} + + + + {/* Folder Management */} + + + + + + Directories to Monitor + + + + + Specify which local directories to scan for files. Use absolute paths. + + + + setNewFolder(e.target.value)} + placeholder="/home/user/Documents" + sx={{ + flexGrow: 1, + '& .MuiOutlinedInput-root': { borderRadius: 2 } + }} + /> + + + + + {formData.watch_folders.map((folder, index) => ( + removeFolder(folder)} + sx={{ + mr: 1, + mb: 1, + borderRadius: 2, + bgcolor: alpha(theme.palette.secondary.main, 0.1), + color: theme.palette.secondary.main, + }} + /> + ))} + + + {/* File Extensions */} + + + + + + File Extensions + + + + + File types to monitor and process with OCR. + + + + setNewExtension(e.target.value)} + placeholder="docx" + sx={{ + flexGrow: 1, + '& .MuiOutlinedInput-root': { borderRadius: 2 } + }} + /> + + + + + {formData.file_extensions.map((extension, index) => ( + removeFileExtension(extension)} + sx={{ + mr: 1, + mb: 1, + borderRadius: 2, + bgcolor: alpha(theme.palette.warning.main, 0.1), + color: theme.palette.warning.main, + }} + /> + ))} + + + + )} + + {formData.source_type === 's3' && ( + + + + + + + + S3 Compatible Storage Configuration + + + + + + Connect to AWS S3, MinIO, or any S3-compatible storage service. + For MinIO, provide the endpoint URL of your server. + + + + + + setFormData({ ...formData, bucket_name: e.target.value })} + placeholder="my-documents-bucket" + sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} + /> + + + setFormData({ ...formData, region: e.target.value })} + placeholder="us-east-1" + sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} + /> + + + + + + setFormData({ ...formData, access_key_id: e.target.value })} + sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} + /> + + + setFormData({ ...formData, secret_access_key: e.target.value })} + sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} + /> + + + + setFormData({ ...formData, endpoint_url: e.target.value })} + placeholder="https://minio.example.com (for MinIO/S3-compatible services)" + helperText="Leave empty for AWS S3, or provide custom endpoint for MinIO/other S3-compatible storage" + sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} + /> + + setFormData({ ...formData, prefix: e.target.value })} + placeholder="documents/" + helperText="Optional prefix to limit scanning to specific object keys" + sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} + /> + + setFormData({ ...formData, auto_sync: e.target.checked })} + /> + } + label={ + + + Enable Automatic Sync + + + Automatically check for new objects on a schedule + + + } + /> + + {formData.auto_sync && ( + setFormData({ ...formData, sync_interval_minutes: parseInt(e.target.value) || 60 })} + inputProps={{ min: 15, max: 1440 }} + helperText="How often to check for new objects (15 min - 24 hours)" + sx={{ '& .MuiOutlinedInput-root': { borderRadius: 2 } }} + /> + )} + + + + {/* Folder Management (prefixes for S3) */} + + + + + + Object Prefixes to Monitor + + + + + Specify which object prefixes (like folders) to scan for files. + + + + setNewFolder(e.target.value)} + placeholder="documents/" + sx={{ + flexGrow: 1, + '& .MuiOutlinedInput-root': { borderRadius: 2 } + }} + /> + + + + + {formData.watch_folders.map((folder, index) => ( + removeFolder(folder)} + sx={{ + mr: 1, + mb: 1, + borderRadius: 2, + bgcolor: alpha(theme.palette.secondary.main, 0.1), + color: theme.palette.secondary.main, + }} + /> + ))} + + + {/* File Extensions */} + + + + + + File Extensions + + + + + File types to sync and process with OCR. + + + + setNewExtension(e.target.value)} + placeholder="docx" + sx={{ + flexGrow: 1, + '& .MuiOutlinedInput-root': { borderRadius: 2 } + }} + /> + + + + + {formData.file_extensions.map((extension, index) => ( + removeFileExtension(extension)} + sx={{ + mr: 1, + mb: 1, + borderRadius: 2, + bgcolor: alpha(theme.palette.warning.main, 0.1), + color: theme.palette.warning.main, + }} + /> + ))} + + + + )} + { > Cancel - {editingSource && formData.source_type === 'webdav' && ( + {(formData.source_type === 'webdav' || formData.source_type === 'local_folder' || formData.source_type === 's3') && (