v1.0.65: chunked upload for large files (>100MB) on Nextcloud
Build & Release APK / build (push) Successful in 12m58s
Build & Release APK / build (push) Successful in 12m58s
Big-file testing found single-PUT uploads 413 above the server's per-request cap (Apache LimitRequestBody / PHP post_max_size / proxy limits). NextcloudProvider now uploads files >chunkSize (100MB) via the dav/uploads chunked API: MKCOL a session, PUT N chunks, then MOVE .file onto the destination (atomic assemble). Bypasses any per-request cap so multi-GB files back up. Verified byte-exact (multi-chunk) against live Nextcloud. SFTP already streams; single-PUT path unchanged for <=100MB.
This commit is contained in:
@@ -13,6 +13,7 @@ import com.syncflow.domain.sync.LocalFileInfo
|
||||
import com.syncflow.domain.sync.SyncDecision
|
||||
import com.syncflow.domain.sync.syncDecide
|
||||
import kotlinx.coroutines.runBlocking
|
||||
import org.junit.Assert.assertArrayEquals
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertNotNull
|
||||
import org.junit.Assert.assertTrue
|
||||
@@ -114,4 +115,27 @@ class NextcloudIntegrationTest {
|
||||
runCatching { p.deleteFile(dir) } // best-effort cleanup of the test folder
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun chunkedUpload_assemblesLargeFileByteExact() = runBlocking {
|
||||
assumeTrue("ncUrl/ncUser/ncPass required", url != null && user != null && pass != null)
|
||||
// Tiny chunk size exercises multi-chunk assembly without needing a multi-GB file.
|
||||
val account = CloudAccount(
|
||||
id = 1, displayName = "IT", email = user, providerType = ProviderType.NEXTCLOUD,
|
||||
credentialJson = """{"username":"$user","password":"$pass"}""", serverUrl = url, port = null,
|
||||
)
|
||||
val p = NextcloudProvider(account, chunkSize = 1L * 1024 * 1024) // 1 MB chunks
|
||||
val dir = "SyncFlowChunk_${System.currentTimeMillis()}"
|
||||
try {
|
||||
p.createDirectory(dir).getOrThrow()
|
||||
val payload = ByteArray(5 * 1024 * 1024 + 7).also { java.util.Random(7).nextBytes(it) } // ~5 MB -> 6 chunks
|
||||
val up = p.uploadFile(ByteArrayInputStream(payload), "$dir/big.bin", payload.size.toLong())
|
||||
assertTrue("chunked upload failed: ${up.exceptionOrNull()}", up.isSuccess)
|
||||
assertEquals(payload.size.toLong(), p.listFiles(dir).getOrThrow().first { it.name == "big.bin" }.sizeBytes)
|
||||
val out = ByteArrayOutputStream(); p.downloadFile("$dir/big.bin", out).getOrThrow()
|
||||
assertArrayEquals("chunk-assembled content must equal the original bytes", payload, out.toByteArray())
|
||||
} finally {
|
||||
runCatching { p.deleteFile(dir) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,13 +2,111 @@ package com.syncflow.data.providers.nextcloud
|
||||
|
||||
import com.syncflow.data.providers.webdav.WebDavProvider
|
||||
import com.syncflow.domain.model.CloudAccount
|
||||
import com.syncflow.domain.model.RemoteFile
|
||||
import kotlinx.coroutines.Dispatchers
|
||||
import kotlinx.coroutines.withContext
|
||||
import okhttp3.MediaType.Companion.toMediaType
|
||||
import okhttp3.Request
|
||||
import okhttp3.RequestBody
|
||||
import okio.BufferedSink
|
||||
import java.io.IOException
|
||||
import java.io.InputStream
|
||||
|
||||
/**
|
||||
* Nextcloud WebDAV provider. Endpoint is /remote.php/dav/files/<username>/.
|
||||
*
|
||||
* Large files are uploaded via Nextcloud's chunked-upload API (/remote.php/dav/uploads/<user>/)
|
||||
* so they bypass per-request size caps (Apache LimitRequestBody, PHP post_max_size, proxy body
|
||||
* limits) that otherwise 413 a single multi-GB PUT. The assembly MOVE is the atomic commit, so
|
||||
* the destination only appears once every chunk is in — no temp-file dance needed for this path.
|
||||
*
|
||||
* @param chunkSize bytes per chunk; files at or below this use the parent's single-PUT path.
|
||||
*/
|
||||
class NextcloudProvider(
|
||||
account: CloudAccount,
|
||||
private val chunkSize: Long = 100L * 1024 * 1024,
|
||||
) : WebDavProvider(account) {
|
||||
|
||||
class NextcloudProvider(account: CloudAccount) : WebDavProvider(account) {
|
||||
// Nextcloud WebDAV endpoint is /remote.php/dav/files/<username>/
|
||||
override val baseUrl: String
|
||||
get() {
|
||||
val server = account.serverUrl?.trimEnd('/') ?: ""
|
||||
val email = account.email ?: "user"
|
||||
return "$server/remote.php/dav/files/$email"
|
||||
}
|
||||
|
||||
private val uploadsBase: String
|
||||
get() {
|
||||
val server = account.serverUrl?.trimEnd('/') ?: ""
|
||||
val email = account.email ?: "user"
|
||||
return "$server/remote.php/dav/uploads/$email"
|
||||
}
|
||||
|
||||
override suspend fun uploadFile(
|
||||
localStream: InputStream,
|
||||
remotePath: String,
|
||||
sizeBytes: Long,
|
||||
onProgress: (Long) -> Unit,
|
||||
): Result<RemoteFile> {
|
||||
if (sizeBytes <= chunkSize) {
|
||||
return super.uploadFile(localStream, remotePath, sizeBytes, onProgress)
|
||||
}
|
||||
return runCatching {
|
||||
withContext(Dispatchers.IO) {
|
||||
val uploadId = "syncflow-${System.currentTimeMillis()}-${(0..999_999).random()}"
|
||||
val dir = "$uploadsBase/$uploadId"
|
||||
mkcol(dir)
|
||||
try {
|
||||
var index = 1
|
||||
var sent = 0L
|
||||
while (sent < sizeBytes) {
|
||||
val len = minOf(chunkSize, sizeBytes - sent)
|
||||
putChunk("$dir/%05d".format(index), localStream, len)
|
||||
sent += len
|
||||
index++
|
||||
onProgress(sent)
|
||||
}
|
||||
// Assemble: MOVE the virtual .file onto the destination (atomic commit).
|
||||
val move = Request.Builder().url("$dir/.file")
|
||||
.method("MOVE", null)
|
||||
.header("Destination", url(remotePath))
|
||||
.header("Overwrite", "T")
|
||||
.header("OC-Total-Length", sizeBytes.toString())
|
||||
.build()
|
||||
client.newCall(move).execute().use { resp ->
|
||||
if (!resp.isSuccessful) throw IOException("Chunk assembly MOVE HTTP ${resp.code}")
|
||||
}
|
||||
} catch (e: Throwable) {
|
||||
runCatching { client.newCall(Request.Builder().url(dir).delete().build()).execute().close() }
|
||||
throw e
|
||||
}
|
||||
getFileMetadata(remotePath).getOrThrow()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun mkcol(url: String) {
|
||||
client.newCall(Request.Builder().url(url).method("MKCOL", null).build()).execute().use {
|
||||
if (!it.isSuccessful && it.code != 405) throw IOException("MKCOL upload session HTTP ${it.code}")
|
||||
}
|
||||
}
|
||||
|
||||
private fun putChunk(url: String, stream: InputStream, len: Long) {
|
||||
val body = object : RequestBody() {
|
||||
override fun contentType() = "application/octet-stream".toMediaType()
|
||||
override fun contentLength() = len
|
||||
override fun writeTo(sink: BufferedSink) {
|
||||
var remaining = len
|
||||
val buf = ByteArray(64 * 1024)
|
||||
while (remaining > 0) {
|
||||
val n = stream.read(buf, 0, minOf(buf.size.toLong(), remaining).toInt())
|
||||
if (n < 0) break
|
||||
sink.write(buf, 0, n)
|
||||
remaining -= n
|
||||
}
|
||||
}
|
||||
}
|
||||
client.newCall(Request.Builder().url(url).put(body).build()).execute().use {
|
||||
if (!it.isSuccessful) throw IOException("Chunk PUT HTTP ${it.code}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user