Compare commits

...

1 Commits

Author SHA1 Message Date
amir fb26e83484 v1.0.74: exclude OS-volatile junk (.thumbnails/.trashed-/.pending-/.sfpart) symmetrically
Build & Release APK / build (push) Successful in 12m50s
The .thumbnails cache was synced then DELETE_REMOTE'd every cycle: exclude
patterns were applied only to the local walk (filename-only), never to the
remote listing, so a previously-uploaded thumbnail looked local-missing and
got mirror-deleted endlessly as Android regenerates the cache.

- isExcludedPath(): path-segment-aware, hardcoded always-ignored set protects
  all existing pairs without a DB migration
- applied symmetrically to remote listing + merged path set (never upload,
  download, or delete an excluded path on either side)
- add .thumbnails to new-pair default excludePatterns
- ExcludePathTest covers cache/trash/pending/sfpart + user patterns
2026-06-07 05:30:00 +00:00
5 changed files with 129 additions and 4 deletions
@@ -110,8 +110,15 @@ class SyncEngine @Inject constructor(
// level (Depth:1) made every file inside a remote subfolder look "missing from remote",
// which on a TWO_WAY/MIRROR pair triggered DELETE_LOCAL and wiped those files off the
// device (data loss). Walk the remote tree so subfolder files are matched, not deleted.
// Exclusions are filtered on BOTH sides. The local walk already drops excluded files,
// but the remote listing did not, so any excluded path that already existed on the
// server (e.g. a previously-uploaded ".thumbnails" entry) looked "local-missing" and
// got DELETE_REMOTE'd every cycle — endless churn as Android regenerates the cache.
// Filtering remote + the merged path set makes excluded paths invisible to the engine:
// never uploaded, downloaded, or deleted on either side.
val remoteFiles = listRemoteFilesRecursive(provider, pair.remotePath)
.associateBy { it.path.removePrefix(pair.remotePath).trimStart('/') }
.filterKeys { !isExcludedPath(it, pair) }
val localFiles = accessor.walkFiles(pair)
// Self-healing: if every known-state path is absent from the current local scan but
@@ -125,7 +132,12 @@ class SyncEngine @Inject constructor(
return performSync(pair, provider, isRetry = true, onProgress = onProgress)
}
val allPaths = (localFiles.keys + remoteFiles.keys + knownStates.keys).toSet()
// knownStates may still hold records for now-excluded paths (e.g. thumbnails uploaded
// by an older build). Drop them from the work set so they aren't acted on; their stale
// state rows are harmless and ignored.
val allPaths = (localFiles.keys + remoteFiles.keys + knownStates.keys)
.filter { !isExcludedPath(it, pair) }
.toSet()
val hasPriorSyncState = knownStates.isNotEmpty()
val semaphore = Semaphore(2) // limit concurrency to be gentle on the server
val uploadedAtomic = AtomicInteger(0)
@@ -409,6 +421,41 @@ internal fun syncDecide(
enum class SyncDecision { UPLOAD, DOWNLOAD, DELETE_LOCAL, DELETE_REMOTE, CONFLICT, SKIP }
/**
* OS-generated, volatile paths that must NEVER sync on any pair, regardless of user exclude
* config. These are matched against every path SEGMENT (directory names included), so an entire
* subtree like "DCIM/.thumbnails/..." is ignored. Android continuously regenerates and evicts
* its thumbnail cache and shuffles files through .trashed-/.pending- staging dirs; syncing them
* produces an endless upload→evict→DELETE_REMOTE→regenerate loop. ".sfpart" is our own atomic-
* write temp suffix and must never be propagated either.
*/
private val ALWAYS_IGNORED_SEGMENTS = setOf(".thumbnails")
private val ALWAYS_IGNORED_PREFIXES = listOf(".trashed-", ".pending-")
private val ALWAYS_IGNORED_SUFFIXES = listOf(".sfpart")
/**
* True if [rel] should be excluded from sync entirely. Applied symmetrically to the local walk,
* the remote listing, and known state so an excluded path is never uploaded, downloaded, or
* deleted. The always-ignored rules run on every segment; user-configured rules
* (skipHiddenFiles, excludePatterns, excludeExtensions) match the filename, mirroring the
* existing local-walk semantics in LocalAccessor.
*/
internal fun isExcludedPath(rel: String, pair: SyncPair): Boolean {
val segments = rel.replace('\\', '/').split('/').filter { it.isNotEmpty() }
if (segments.isEmpty()) return false
for (seg in segments) {
if (seg in ALWAYS_IGNORED_SEGMENTS) return true
if (ALWAYS_IGNORED_PREFIXES.any { seg.startsWith(it) }) return true
if (ALWAYS_IGNORED_SUFFIXES.any { seg.endsWith(it) }) return true
}
val fileName = segments.last()
if (pair.skipHiddenFiles && fileName.startsWith('.')) return true
if (pair.excludePatterns.any { pat -> fileName.matches(globToRegex(pat)) }) return true
val ext = fileName.substringAfterLast('.', "").lowercase()
if (pair.excludeExtensions.any { ext == it.lowercase().trimStart('.') }) return true
return false
}
/**
* True if a relative sync path is unsafe to act on — empty, absolute, or containing a ".."
* segment that would let a hostile remote escape the sync root via path traversal. Applied to
@@ -47,7 +47,7 @@ data class AddPairUiState(
val chargingOnly: Boolean = false,
val minBatteryPct: Int = 0,
// ── File filters ─────────────────────────────────────────────────────────
val excludePatterns: String = ".DS_Store\n*.tmp\n.nomedia\nThumbs.db",
val excludePatterns: String = ".DS_Store\n*.tmp\n.nomedia\nThumbs.db\n.thumbnails",
val includeExtensions: String = "",
val excludeExtensions: String = "",
val skipHiddenFiles: Boolean = true,
@@ -0,0 +1,78 @@
package com.syncflow.domain.sync
import com.syncflow.domain.model.ConflictStrategy
import com.syncflow.domain.model.DeleteBehavior
import com.syncflow.domain.model.ScheduleType
import com.syncflow.domain.model.SyncDirection
import com.syncflow.domain.model.SyncPair
import com.syncflow.domain.model.SyncStatus
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
/**
* Exclusion symmetry: paths excluded from the local walk must also be excluded from the remote
* listing and the merged work set, so a previously-uploaded excluded file is never DELETE_REMOTE'd
* in an endless churn loop (the ".thumbnails" cache regression).
*/
class ExcludePathTest {
private fun pair(
excludePatterns: List<String> = emptyList(),
excludeExtensions: List<String> = emptyList(),
skipHiddenFiles: Boolean = false,
) = SyncPair(
id = 1, name = "t", localPath = "/l", remotePath = "/r", accountId = 1,
syncDirection = SyncDirection.TWO_WAY,
conflictStrategy = ConflictStrategy.KEEP_NEWEST,
deleteBehavior = DeleteBehavior.MIRROR,
recursive = true,
scheduleType = ScheduleType.MANUAL, scheduleIntervalMinutes = 0,
scheduleDailyTime = null, scheduleWeekdays = 0,
wifiOnly = false, wifiSsid = "", chargingOnly = false, minBatteryPct = 0,
excludePatterns = excludePatterns, includeExtensions = emptyList(),
excludeExtensions = excludeExtensions, skipHiddenFiles = skipHiddenFiles,
minFileSizeKb = 0, maxFileSizeKb = 0,
notifyOnComplete = false, notifyOnError = false,
isEnabled = true, lastSyncAt = null, lastSyncResult = SyncStatus.IDLE,
pendingConflicts = 0,
)
@Test fun `thumbnails cache is always excluded regardless of config`() {
val p = pair()
assertTrue(isExcludedPath("DCIM/.thumbnails/123.jpg", p))
assertTrue(isExcludedPath("Pictures/.thumbnails/1000020397.jpg", p))
assertTrue(isExcludedPath(".thumbnails/x.jpg", p))
}
@Test fun `android trash and pending staging dirs are excluded`() {
val p = pair()
assertTrue(isExcludedPath("DCIM/Camera/.trashed-1700000000-IMG_0001.jpg", p))
assertTrue(isExcludedPath("DCIM/.pending-1700000000-VID_0001.mp4", p))
}
@Test fun `our own atomic-write temp files are excluded`() {
val p = pair()
assertTrue(isExcludedPath("Download/.movie.mp4.sfpart", p))
assertTrue(isExcludedPath("a/b/.photo.jpg.sfpart", p))
}
@Test fun `normal media files are not excluded`() {
val p = pair()
assertFalse(isExcludedPath("DCIM/Camera/IMG_0001.jpg", p))
assertFalse(isExcludedPath("Pictures/cat.png", p))
assertFalse(isExcludedPath("اصفهان/20171023.jpg", p)) // unicode folder, real data
}
@Test fun `user exclude patterns and extensions still apply on filename`() {
assertTrue(isExcludedPath("a/Thumbs.db", pair(excludePatterns = listOf("Thumbs.db"))))
assertTrue(isExcludedPath("a/note.tmp", pair(excludePatterns = listOf("*.tmp"))))
assertTrue(isExcludedPath("a/data.log", pair(excludeExtensions = listOf("log"))))
assertFalse(isExcludedPath("a/keep.jpg", pair(excludeExtensions = listOf("log"))))
}
@Test fun `skipHiddenFiles excludes dotfiles by filename only`() {
assertTrue(isExcludedPath("a/.hidden", pair(skipHiddenFiles = true)))
assertFalse(isExcludedPath("a/.hidden", pair(skipHiddenFiles = false)))
}
}
+2 -2
View File
@@ -1,2 +1,2 @@
VERSION_NAME=1.0.73
VERSION_CODE=73
VERSION_NAME=1.0.74
VERSION_CODE=74