From 48bffeb1fd92864efc57acb36131cf1664091691 Mon Sep 17 00:00:00 2001 From: perf3ct Date: Sun, 27 Jul 2025 00:07:17 +0000 Subject: [PATCH] fix(tests): resolve issue with new url management test --- .github/workflows/.test-integration.yml.swp | Bin 16384 -> 0 bytes pr_description.md | 130 -------------------- 2 files changed, 130 deletions(-) delete mode 100644 .github/workflows/.test-integration.yml.swp delete mode 100644 pr_description.md diff --git a/.github/workflows/.test-integration.yml.swp b/.github/workflows/.test-integration.yml.swp deleted file mode 100644 index 4c9d610643e416deb27c70725326661a0af286f4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeI2Ta4UR8ONP)X`wV!r4`~Pec42NiSW#BF0@RvQD%21$&yWXIy0MqNHlotGc(S5 zY{x#H%_y5xLIvJ6lMLz&vvW#?^53X+AhU&94;Q;?OLhS zPdC=9>f@C&r_Vn23t7x>^x~6Ggly2AXgw)AXgw)AXgw) zAXgw);Qw9$?(SBeLM?X3HMt!B?i%@hTYP^g{vC{j$G7=Uu0XCpu0XCpu0XCpu0XCp zu0XCpu0XCpu0XE9TTlVbP?QrW*D>J$!1;e-|NrT|it-wG75o`|28gWrK)f*atQ;LBhO)WH!j4ITpjd{9w-16~5pfv3PV;DdAE z2sjM>{vJhn9b5-Z@FDQ?2NdOH@H}`1a8Lz@!6V?-yA|aX@FKVlIM@Im10Mmq!Rzl* zlwX5ifg9ji@C04{=A@aK0Z%1!VS@Huc9oB~teA8&^);3e<^_%_%EJ@5qh7o0OU!42>|a2Xh2 z12n)CxQX`t3b+JBTQ7n)e^KUQde@<;I`jI>qw4BPiT2Ox8mH)6kAiC&b5n|(!fey0 zNo`9J=)zIrU1n#ZMfp~0we?zw;_Ta+X)%MiLq69$POj&;%(Di2ozqN?)YNvoSdjP~g+hU5_nXB- z^CR-L%}h2#EhxP?@+``{7JhCc6Sya>QOSBVaXs= ziMHcgm29oYN+jwZRz6l1P1|Cj67V(B@LH)XZJaF8g7oNIxv_kzd18HSrGgGpC~j$7 z?@Hkt)wT8VO0!ySoD#u-YdM8t2k7 zZOzMc9SU1Rj8f(m93Rb~hPkSd3`wS{RN0*-5zXAy{O$?NO8)H3fdj>5QC=vhZ_dn# zH%Xa#_{7cenCLftIt~aIod?uESj8^>cDD0qHY8mTa@#=GdwVMiD~7Ok0f%fM$S zC#E@kj^`J3P4BYf{ljtOMEhm&IL?O*(_=d7xHI`Z)-iqV4c@#)!<@$JKGGh;suHei z#X{H-iuf9FazK)?6dDPa%wiOd(*H=a@YNWhkqBwGk7PHvW7*oE=mq#Lk~@y2cQw;K zj=7;_B}+@v26e^*muiTa<|68+TN;)n@vr1^bgwsXjW(TEl1Jj#GFxF>6${}=kh|GY zb;oX-osj^fv6#yp+tjrQ=@5*$60-#Q&xGJK3`J61tQy%8Wy4Lb-7(x?!5zcf-ksC? z?w!LKCcHb8FB9emhC`amRm`wh2+?5JSEz&A#|d0R<4Kc}*iEF9L(;;laX$XQ=d4Et zbD3>0TQ`|6H+iz?_nbb~k8sN)Rv~JO=}BywnC%B1qvh4`abNTBRNG|OMlqd<3|Ts1 zQ?yAdh-FMHdvt-Or>WPc^vUpFg_fpoq7QCw6Ze9&GEyFy4-Ekhk@MeF}#9~q}3rThjCKz z*|gW!ELxbK7_RpWayw1a4|jjV{2e!(*-Laxo`F?5a`@1}lK3->iSfx%Y#LtG^?KtZ zHcicyrBb}fm=szq*X!rjYp@8S#d%Vk4eY|k!ZX_=v!`{iQN-utyn{8gwD4G*&+tij zFLorRj_^ocG`Fuh7)wDbuz8?b7-X0rvENNUGFQbTB)I6bqGa;cC4J*13kF0~gHrT! zJT)hTP;nqKUoO{9t~VQ%+L`9^`kF9YG;2#6_)1u#xZEoBFqca==oVgh+wwFU>uIR9 z3xlNTu}cBUm}tM#ls=Ym)Ll`cJ@#r^Won2jE5iOCz+Sxon^xHW$@%^dVZXlsu7fXt z&x22bRqy~1cKy%5kHD9}b+8G}gO7rjVW)o$d=&(s4W_|va0B*w8+;1Pfm^W8h3)<= za1C4li(ntP58Q%n{ul65@B{D-a1ERQ9|Ru&e?$3S1+Rebf~UbnAnJ7<AUL&qruQ%Bv2M>@5D{;(iKEa;af$uzH=m`IAvI(7?8}2!yePNwD zZd{@WEVS*s){vse0JlXI|*o6i1egt_jZM+D5(6Tq-Z0Zq&-lm8_#V3|m`= z``KWvK=!D-wOr8U>T2YDo_OXFx4r0_jS4E8wRME&Oj!EzQ`uonB!}FygFA9|g>D2j zw7O{Kzzi(dHq$s-5^e~XC2GJ;%g0mOQAMla58Mu_ivPo4-WArDzbGF9pv<&QunS#M zD>OALmvTGx+;7L)%uOM-4|7&hBkF?{yFYXuzy#AbOXK~t7M=8jh4mx3>Oxh|5#5DsN1U{UGw{Cw1|=)B diff --git a/pr_description.md b/pr_description.md deleted file mode 100644 index 3c6dde9..0000000 --- a/pr_description.md +++ /dev/null @@ -1,130 +0,0 @@ -## Summary - -This PR implements comprehensive smart sync functionality for WebDAV sources and fixes critical path handling issues that were causing 404 errors with Nextcloud servers. - -## Key Features - -### 🧠 Smart Sync Implementation -- **Directory ETag Tracking**: Tracks ETags for all directories at infinite depth to detect changes -- **Bulk Database Operations**: Fetches all directory ETags in single query to avoid database round trips -- **Intelligent Sync Strategies**: - - **SkipSync**: No changes detected, sync skipped entirely - - **TargetedScan**: Scans only changed directories for efficiency - - **FullDeepScan**: Complete rescan when many changes detected or first-time sync -- **Smart Decision Making**: Automatically chooses optimal strategy based on change scope - -### 🔧 WebDAV Path Management Overhaul -- **Centralized URL Management**: New `url_management.rs` module handles all WebDAV path operations -- **Fixed Nextcloud URL Doubling**: Resolved critical issue where "remote.php" was incorrectly appended causing 404s -- **Multi-Server Support**: Proper path handling for Nextcloud, ownCloud, and generic WebDAV servers -- **Path Field Migration**: Added `relative_path` and `full_path` fields while maintaining backward compatibility - -### 📊 Enhanced Data Model -```rust -pub struct FileIngestionInfo { - /// Clean relative path from WebDAV root (e.g., "/Photos/image.jpg") - pub relative_path: String, - /// Full WebDAV path as returned by server (e.g., "/remote.php/dav/files/user/Photos/image.jpg") - pub full_path: String, - /// Legacy field - deprecated, use relative_path instead - #[deprecated(note = "Use relative_path instead for new code")] - pub path: String, - // ... other fields -} -``` - -## Technical Implementation - -### Smart Sync Service Architecture -- **SmartSyncService**: Main service for intelligent sync evaluation and execution -- **SmartSyncDecision**: Enum determining whether sync is needed -- **SmartSyncStrategy**: Enum defining how sync should be performed -- **Bulk Directory Fetching**: Single database query for all directory ETags -- **Recursive Directory Tracking**: Tracks subdirectories at all depth levels - -### URL Management System -- **Server-Specific Logic**: Handles different WebDAV server path formats -- **Path Conversion**: Converts between full WebDAV paths and relative paths -- **URL Construction**: Builds correct URLs for file operations -- **Backward Compatibility**: Maintains existing functionality during migration - -## Problem Solved - -### Original Issue -User discovered that only one folder's ETag was being tracked in the database instead of all subfolders in the WebDAV directory hierarchy. This meant: -- Only root directory changes were detected -- Subdirectory changes were missed -- No performance optimization for unchanged directory trees -- Unnecessary full scans on every sync - -### WebDAV Path Issue -Nextcloud users experienced 404 errors due to URL doubling: -- **Before**: `https://server.com/remote.php/dav/files/user/remote.php/dav/files/user/Photos/image.jpg` -- **After**: `https://server.com/remote.php/dav/files/user/Photos/image.jpg` - -## Changes Made - -### Core Implementation -- ✅ Created `SmartSyncService` with comprehensive directory ETag tracking -- ✅ Implemented bulk database operations for performance -- ✅ Added smart sync as default behavior (not optional) -- ✅ Created centralized `url_management.rs` module -- ✅ Updated XML parser to use new path management -- ✅ Migrated FileIngestionInfo to new field structure - -### Testing Infrastructure -- ✅ **22+ test files updated** with new field requirements -- ✅ **Comprehensive test coverage** for all smart sync scenarios -- ✅ **Integration tests** for first-time sync, directory changes, deep scans -- ✅ **Unit tests** for decision logic, ETag comparison, strategy selection -- ✅ **Path handling tests** to prevent regression of URL doubling issue -- ✅ **Database connection pool** fixes for test environment - -### Backward Compatibility -- ✅ Deprecated `path` field maintained with warnings -- ✅ Existing code continues to work during migration -- ✅ Clean migration path to new field structure - -## Test Results - -```bash -✅ All library tests compile: cargo test --lib --no-run -✅ Integration tests compile successfully -✅ No compilation errors (only expected deprecation warnings) -✅ Comprehensive test coverage for all scenarios -``` - -## Performance Impact - -### Before -- Database query for each directory check -- Full scan on every sync regardless of changes -- Inefficient for large directory structures -- URL construction errors causing failed requests - -### After -- Single bulk query for all directory ETags -- Smart sync skips unchanged directory trees -- Targeted scans for minimal changes -- Correct URL construction for all server types -- Significant performance improvement for large WebDAV folders - -## Migration Notes - -- **Smart sync is now the default behavior** (not optional) -- **Deep scans reset all directory ETags** at all levels for fresh baselines -- **Path field deprecation** - new code should use `relative_path` and `full_path` -- **URL management centralized** - prevents future path handling issues - -## Future Benefits - -This implementation provides the foundation for: -- More efficient WebDAV synchronization -- Better support for large directory structures -- Reliable path handling across different WebDAV servers -- Extensible smart sync strategies -- Improved user experience with faster syncs - -🤖 Generated with [Claude Code](https://claude.ai/code) - -Co-Authored-By: Claude \ No newline at end of file