From fb26e83484d0fb83017853775c8531f4faf75265 Mon Sep 17 00:00:00 2001 From: Amir Date: Sun, 7 Jun 2026 05:30:00 +0000 Subject: [PATCH] v1.0.74: exclude OS-volatile junk (.thumbnails/.trashed-/.pending-/.sfpart) symmetrically The .thumbnails cache was synced then DELETE_REMOTE'd every cycle: exclude patterns were applied only to the local walk (filename-only), never to the remote listing, so a previously-uploaded thumbnail looked local-missing and got mirror-deleted endlessly as Android regenerates the cache. - isExcludedPath(): path-segment-aware, hardcoded always-ignored set protects all existing pairs without a DB migration - applied symmetrically to remote listing + merged path set (never upload, download, or delete an excluded path on either side) - add .thumbnails to new-pair default excludePatterns - ExcludePathTest covers cache/trash/pending/sfpart + user patterns --- ...otlin-compiler-17831070670597191982.salive | 0 .../com/syncflow/domain/sync/SyncEngine.kt | 49 +++++++++++- .../syncflow/ui/addpair/AddPairViewModel.kt | 2 +- .../syncflow/domain/sync/ExcludePathTest.kt | 78 +++++++++++++++++++ version.properties | 4 +- 5 files changed, 129 insertions(+), 4 deletions(-) delete mode 100644 .kotlin/sessions/kotlin-compiler-17831070670597191982.salive create mode 100644 app/src/test/kotlin/com/syncflow/domain/sync/ExcludePathTest.kt diff --git a/.kotlin/sessions/kotlin-compiler-17831070670597191982.salive b/.kotlin/sessions/kotlin-compiler-17831070670597191982.salive deleted file mode 100644 index e69de29..0000000 diff --git a/app/src/main/kotlin/com/syncflow/domain/sync/SyncEngine.kt b/app/src/main/kotlin/com/syncflow/domain/sync/SyncEngine.kt index 3f90d99..c65c1b5 100644 --- a/app/src/main/kotlin/com/syncflow/domain/sync/SyncEngine.kt +++ b/app/src/main/kotlin/com/syncflow/domain/sync/SyncEngine.kt @@ -110,8 +110,15 @@ class SyncEngine @Inject constructor( // level (Depth:1) made every file inside a remote subfolder look "missing from remote", // which on a TWO_WAY/MIRROR pair triggered DELETE_LOCAL and wiped those files off the // device (data loss). Walk the remote tree so subfolder files are matched, not deleted. + // Exclusions are filtered on BOTH sides. The local walk already drops excluded files, + // but the remote listing did not, so any excluded path that already existed on the + // server (e.g. a previously-uploaded ".thumbnails" entry) looked "local-missing" and + // got DELETE_REMOTE'd every cycle — endless churn as Android regenerates the cache. + // Filtering remote + the merged path set makes excluded paths invisible to the engine: + // never uploaded, downloaded, or deleted on either side. val remoteFiles = listRemoteFilesRecursive(provider, pair.remotePath) .associateBy { it.path.removePrefix(pair.remotePath).trimStart('/') } + .filterKeys { !isExcludedPath(it, pair) } val localFiles = accessor.walkFiles(pair) // Self-healing: if every known-state path is absent from the current local scan but @@ -125,7 +132,12 @@ class SyncEngine @Inject constructor( return performSync(pair, provider, isRetry = true, onProgress = onProgress) } - val allPaths = (localFiles.keys + remoteFiles.keys + knownStates.keys).toSet() + // knownStates may still hold records for now-excluded paths (e.g. thumbnails uploaded + // by an older build). Drop them from the work set so they aren't acted on; their stale + // state rows are harmless and ignored. + val allPaths = (localFiles.keys + remoteFiles.keys + knownStates.keys) + .filter { !isExcludedPath(it, pair) } + .toSet() val hasPriorSyncState = knownStates.isNotEmpty() val semaphore = Semaphore(2) // limit concurrency to be gentle on the server val uploadedAtomic = AtomicInteger(0) @@ -409,6 +421,41 @@ internal fun syncDecide( enum class SyncDecision { UPLOAD, DOWNLOAD, DELETE_LOCAL, DELETE_REMOTE, CONFLICT, SKIP } +/** + * OS-generated, volatile paths that must NEVER sync on any pair, regardless of user exclude + * config. These are matched against every path SEGMENT (directory names included), so an entire + * subtree like "DCIM/.thumbnails/..." is ignored. Android continuously regenerates and evicts + * its thumbnail cache and shuffles files through .trashed-/.pending- staging dirs; syncing them + * produces an endless upload→evict→DELETE_REMOTE→regenerate loop. ".sfpart" is our own atomic- + * write temp suffix and must never be propagated either. + */ +private val ALWAYS_IGNORED_SEGMENTS = setOf(".thumbnails") +private val ALWAYS_IGNORED_PREFIXES = listOf(".trashed-", ".pending-") +private val ALWAYS_IGNORED_SUFFIXES = listOf(".sfpart") + +/** + * True if [rel] should be excluded from sync entirely. Applied symmetrically to the local walk, + * the remote listing, and known state so an excluded path is never uploaded, downloaded, or + * deleted. The always-ignored rules run on every segment; user-configured rules + * (skipHiddenFiles, excludePatterns, excludeExtensions) match the filename, mirroring the + * existing local-walk semantics in LocalAccessor. + */ +internal fun isExcludedPath(rel: String, pair: SyncPair): Boolean { + val segments = rel.replace('\\', '/').split('/').filter { it.isNotEmpty() } + if (segments.isEmpty()) return false + for (seg in segments) { + if (seg in ALWAYS_IGNORED_SEGMENTS) return true + if (ALWAYS_IGNORED_PREFIXES.any { seg.startsWith(it) }) return true + if (ALWAYS_IGNORED_SUFFIXES.any { seg.endsWith(it) }) return true + } + val fileName = segments.last() + if (pair.skipHiddenFiles && fileName.startsWith('.')) return true + if (pair.excludePatterns.any { pat -> fileName.matches(globToRegex(pat)) }) return true + val ext = fileName.substringAfterLast('.', "").lowercase() + if (pair.excludeExtensions.any { ext == it.lowercase().trimStart('.') }) return true + return false +} + /** * True if a relative sync path is unsafe to act on — empty, absolute, or containing a ".." * segment that would let a hostile remote escape the sync root via path traversal. Applied to diff --git a/app/src/main/kotlin/com/syncflow/ui/addpair/AddPairViewModel.kt b/app/src/main/kotlin/com/syncflow/ui/addpair/AddPairViewModel.kt index 349b6a6..9ffaf7b 100644 --- a/app/src/main/kotlin/com/syncflow/ui/addpair/AddPairViewModel.kt +++ b/app/src/main/kotlin/com/syncflow/ui/addpair/AddPairViewModel.kt @@ -47,7 +47,7 @@ data class AddPairUiState( val chargingOnly: Boolean = false, val minBatteryPct: Int = 0, // ── File filters ───────────────────────────────────────────────────────── - val excludePatterns: String = ".DS_Store\n*.tmp\n.nomedia\nThumbs.db", + val excludePatterns: String = ".DS_Store\n*.tmp\n.nomedia\nThumbs.db\n.thumbnails", val includeExtensions: String = "", val excludeExtensions: String = "", val skipHiddenFiles: Boolean = true, diff --git a/app/src/test/kotlin/com/syncflow/domain/sync/ExcludePathTest.kt b/app/src/test/kotlin/com/syncflow/domain/sync/ExcludePathTest.kt new file mode 100644 index 0000000..a3fe4f6 --- /dev/null +++ b/app/src/test/kotlin/com/syncflow/domain/sync/ExcludePathTest.kt @@ -0,0 +1,78 @@ +package com.syncflow.domain.sync + +import com.syncflow.domain.model.ConflictStrategy +import com.syncflow.domain.model.DeleteBehavior +import com.syncflow.domain.model.ScheduleType +import com.syncflow.domain.model.SyncDirection +import com.syncflow.domain.model.SyncPair +import com.syncflow.domain.model.SyncStatus +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * Exclusion symmetry: paths excluded from the local walk must also be excluded from the remote + * listing and the merged work set, so a previously-uploaded excluded file is never DELETE_REMOTE'd + * in an endless churn loop (the ".thumbnails" cache regression). + */ +class ExcludePathTest { + + private fun pair( + excludePatterns: List = emptyList(), + excludeExtensions: List = emptyList(), + skipHiddenFiles: Boolean = false, + ) = SyncPair( + id = 1, name = "t", localPath = "/l", remotePath = "/r", accountId = 1, + syncDirection = SyncDirection.TWO_WAY, + conflictStrategy = ConflictStrategy.KEEP_NEWEST, + deleteBehavior = DeleteBehavior.MIRROR, + recursive = true, + scheduleType = ScheduleType.MANUAL, scheduleIntervalMinutes = 0, + scheduleDailyTime = null, scheduleWeekdays = 0, + wifiOnly = false, wifiSsid = "", chargingOnly = false, minBatteryPct = 0, + excludePatterns = excludePatterns, includeExtensions = emptyList(), + excludeExtensions = excludeExtensions, skipHiddenFiles = skipHiddenFiles, + minFileSizeKb = 0, maxFileSizeKb = 0, + notifyOnComplete = false, notifyOnError = false, + isEnabled = true, lastSyncAt = null, lastSyncResult = SyncStatus.IDLE, + pendingConflicts = 0, + ) + + @Test fun `thumbnails cache is always excluded regardless of config`() { + val p = pair() + assertTrue(isExcludedPath("DCIM/.thumbnails/123.jpg", p)) + assertTrue(isExcludedPath("Pictures/.thumbnails/1000020397.jpg", p)) + assertTrue(isExcludedPath(".thumbnails/x.jpg", p)) + } + + @Test fun `android trash and pending staging dirs are excluded`() { + val p = pair() + assertTrue(isExcludedPath("DCIM/Camera/.trashed-1700000000-IMG_0001.jpg", p)) + assertTrue(isExcludedPath("DCIM/.pending-1700000000-VID_0001.mp4", p)) + } + + @Test fun `our own atomic-write temp files are excluded`() { + val p = pair() + assertTrue(isExcludedPath("Download/.movie.mp4.sfpart", p)) + assertTrue(isExcludedPath("a/b/.photo.jpg.sfpart", p)) + } + + @Test fun `normal media files are not excluded`() { + val p = pair() + assertFalse(isExcludedPath("DCIM/Camera/IMG_0001.jpg", p)) + assertFalse(isExcludedPath("Pictures/cat.png", p)) + assertFalse(isExcludedPath("اصفهان/20171023.jpg", p)) // unicode folder, real data + } + + @Test fun `user exclude patterns and extensions still apply on filename`() { + assertTrue(isExcludedPath("a/Thumbs.db", pair(excludePatterns = listOf("Thumbs.db")))) + assertTrue(isExcludedPath("a/note.tmp", pair(excludePatterns = listOf("*.tmp")))) + assertTrue(isExcludedPath("a/data.log", pair(excludeExtensions = listOf("log")))) + assertFalse(isExcludedPath("a/keep.jpg", pair(excludeExtensions = listOf("log")))) + } + + @Test fun `skipHiddenFiles excludes dotfiles by filename only`() { + assertTrue(isExcludedPath("a/.hidden", pair(skipHiddenFiles = true))) + assertFalse(isExcludedPath("a/.hidden", pair(skipHiddenFiles = false))) + } +} diff --git a/version.properties b/version.properties index 5f634f7..c0601a3 100644 --- a/version.properties +++ b/version.properties @@ -1,2 +1,2 @@ -VERSION_NAME=1.0.73 -VERSION_CODE=73 +VERSION_NAME=1.0.74 +VERSION_CODE=74