v1.0.65: chunked upload for large files (>100MB) on Nextcloud
Build & Release APK / build (push) Successful in 12m58s

Big-file testing found single-PUT uploads 413 above the server's per-request
cap (Apache LimitRequestBody / PHP post_max_size / proxy limits). NextcloudProvider
now uploads files >chunkSize (100MB) via the dav/uploads chunked API: MKCOL a
session, PUT N chunks, then MOVE .file onto the destination (atomic assemble).
Bypasses any per-request cap so multi-GB files back up. Verified byte-exact
(multi-chunk) against live Nextcloud. SFTP already streams; single-PUT path
unchanged for <=100MB.
This commit is contained in:
2026-06-05 15:45:47 +00:00
parent f90d84e1fc
commit a348c43c66
3 changed files with 126 additions and 4 deletions
@@ -13,6 +13,7 @@ import com.syncflow.domain.sync.LocalFileInfo
import com.syncflow.domain.sync.SyncDecision
import com.syncflow.domain.sync.syncDecide
import kotlinx.coroutines.runBlocking
import org.junit.Assert.assertArrayEquals
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertTrue
@@ -114,4 +115,27 @@ class NextcloudIntegrationTest {
runCatching { p.deleteFile(dir) } // best-effort cleanup of the test folder
}
}
@Test
fun chunkedUpload_assemblesLargeFileByteExact() = runBlocking {
assumeTrue("ncUrl/ncUser/ncPass required", url != null && user != null && pass != null)
// Tiny chunk size exercises multi-chunk assembly without needing a multi-GB file.
val account = CloudAccount(
id = 1, displayName = "IT", email = user, providerType = ProviderType.NEXTCLOUD,
credentialJson = """{"username":"$user","password":"$pass"}""", serverUrl = url, port = null,
)
val p = NextcloudProvider(account, chunkSize = 1L * 1024 * 1024) // 1 MB chunks
val dir = "SyncFlowChunk_${System.currentTimeMillis()}"
try {
p.createDirectory(dir).getOrThrow()
val payload = ByteArray(5 * 1024 * 1024 + 7).also { java.util.Random(7).nextBytes(it) } // ~5 MB -> 6 chunks
val up = p.uploadFile(ByteArrayInputStream(payload), "$dir/big.bin", payload.size.toLong())
assertTrue("chunked upload failed: ${up.exceptionOrNull()}", up.isSuccess)
assertEquals(payload.size.toLong(), p.listFiles(dir).getOrThrow().first { it.name == "big.bin" }.sizeBytes)
val out = ByteArrayOutputStream(); p.downloadFile("$dir/big.bin", out).getOrThrow()
assertArrayEquals("chunk-assembled content must equal the original bytes", payload, out.toByteArray())
} finally {
runCatching { p.deleteFile(dir) }
}
}
}
@@ -2,13 +2,111 @@ package com.syncflow.data.providers.nextcloud
import com.syncflow.data.providers.webdav.WebDavProvider
import com.syncflow.domain.model.CloudAccount
import com.syncflow.domain.model.RemoteFile
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.Request
import okhttp3.RequestBody
import okio.BufferedSink
import java.io.IOException
import java.io.InputStream
/**
* Nextcloud WebDAV provider. Endpoint is /remote.php/dav/files/<username>/.
*
* Large files are uploaded via Nextcloud's chunked-upload API (/remote.php/dav/uploads/<user>/)
* so they bypass per-request size caps (Apache LimitRequestBody, PHP post_max_size, proxy body
* limits) that otherwise 413 a single multi-GB PUT. The assembly MOVE is the atomic commit, so
* the destination only appears once every chunk is in — no temp-file dance needed for this path.
*
* @param chunkSize bytes per chunk; files at or below this use the parent's single-PUT path.
*/
class NextcloudProvider(
account: CloudAccount,
private val chunkSize: Long = 100L * 1024 * 1024,
) : WebDavProvider(account) {
class NextcloudProvider(account: CloudAccount) : WebDavProvider(account) {
// Nextcloud WebDAV endpoint is /remote.php/dav/files/<username>/
override val baseUrl: String
get() {
val server = account.serverUrl?.trimEnd('/') ?: ""
val email = account.email ?: "user"
return "$server/remote.php/dav/files/$email"
}
private val uploadsBase: String
get() {
val server = account.serverUrl?.trimEnd('/') ?: ""
val email = account.email ?: "user"
return "$server/remote.php/dav/uploads/$email"
}
override suspend fun uploadFile(
localStream: InputStream,
remotePath: String,
sizeBytes: Long,
onProgress: (Long) -> Unit,
): Result<RemoteFile> {
if (sizeBytes <= chunkSize) {
return super.uploadFile(localStream, remotePath, sizeBytes, onProgress)
}
return runCatching {
withContext(Dispatchers.IO) {
val uploadId = "syncflow-${System.currentTimeMillis()}-${(0..999_999).random()}"
val dir = "$uploadsBase/$uploadId"
mkcol(dir)
try {
var index = 1
var sent = 0L
while (sent < sizeBytes) {
val len = minOf(chunkSize, sizeBytes - sent)
putChunk("$dir/%05d".format(index), localStream, len)
sent += len
index++
onProgress(sent)
}
// Assemble: MOVE the virtual .file onto the destination (atomic commit).
val move = Request.Builder().url("$dir/.file")
.method("MOVE", null)
.header("Destination", url(remotePath))
.header("Overwrite", "T")
.header("OC-Total-Length", sizeBytes.toString())
.build()
client.newCall(move).execute().use { resp ->
if (!resp.isSuccessful) throw IOException("Chunk assembly MOVE HTTP ${resp.code}")
}
} catch (e: Throwable) {
runCatching { client.newCall(Request.Builder().url(dir).delete().build()).execute().close() }
throw e
}
getFileMetadata(remotePath).getOrThrow()
}
}
}
private fun mkcol(url: String) {
client.newCall(Request.Builder().url(url).method("MKCOL", null).build()).execute().use {
if (!it.isSuccessful && it.code != 405) throw IOException("MKCOL upload session HTTP ${it.code}")
}
}
private fun putChunk(url: String, stream: InputStream, len: Long) {
val body = object : RequestBody() {
override fun contentType() = "application/octet-stream".toMediaType()
override fun contentLength() = len
override fun writeTo(sink: BufferedSink) {
var remaining = len
val buf = ByteArray(64 * 1024)
while (remaining > 0) {
val n = stream.read(buf, 0, minOf(buf.size.toLong(), remaining).toInt())
if (n < 0) break
sink.write(buf, 0, n)
remaining -= n
}
}
}
client.newCall(Request.Builder().url(url).put(body).build()).execute().use {
if (!it.isSuccessful) throw IOException("Chunk PUT HTTP ${it.code}")
}
}
}
+2 -2
View File
@@ -1,2 +1,2 @@
VERSION_NAME=1.0.64
VERSION_CODE=65
VERSION_NAME=1.0.65
VERSION_CODE=66