v1.0.65: chunked upload for large files (>100MB) on Nextcloud
Build & Release APK / build (push) Successful in 12m58s
Build & Release APK / build (push) Successful in 12m58s
Big-file testing found single-PUT uploads 413 above the server's per-request cap (Apache LimitRequestBody / PHP post_max_size / proxy limits). NextcloudProvider now uploads files >chunkSize (100MB) via the dav/uploads chunked API: MKCOL a session, PUT N chunks, then MOVE .file onto the destination (atomic assemble). Bypasses any per-request cap so multi-GB files back up. Verified byte-exact (multi-chunk) against live Nextcloud. SFTP already streams; single-PUT path unchanged for <=100MB.
This commit is contained in:
@@ -13,6 +13,7 @@ import com.syncflow.domain.sync.LocalFileInfo
|
|||||||
import com.syncflow.domain.sync.SyncDecision
|
import com.syncflow.domain.sync.SyncDecision
|
||||||
import com.syncflow.domain.sync.syncDecide
|
import com.syncflow.domain.sync.syncDecide
|
||||||
import kotlinx.coroutines.runBlocking
|
import kotlinx.coroutines.runBlocking
|
||||||
|
import org.junit.Assert.assertArrayEquals
|
||||||
import org.junit.Assert.assertEquals
|
import org.junit.Assert.assertEquals
|
||||||
import org.junit.Assert.assertNotNull
|
import org.junit.Assert.assertNotNull
|
||||||
import org.junit.Assert.assertTrue
|
import org.junit.Assert.assertTrue
|
||||||
@@ -114,4 +115,27 @@ class NextcloudIntegrationTest {
|
|||||||
runCatching { p.deleteFile(dir) } // best-effort cleanup of the test folder
|
runCatching { p.deleteFile(dir) } // best-effort cleanup of the test folder
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun chunkedUpload_assemblesLargeFileByteExact() = runBlocking {
|
||||||
|
assumeTrue("ncUrl/ncUser/ncPass required", url != null && user != null && pass != null)
|
||||||
|
// Tiny chunk size exercises multi-chunk assembly without needing a multi-GB file.
|
||||||
|
val account = CloudAccount(
|
||||||
|
id = 1, displayName = "IT", email = user, providerType = ProviderType.NEXTCLOUD,
|
||||||
|
credentialJson = """{"username":"$user","password":"$pass"}""", serverUrl = url, port = null,
|
||||||
|
)
|
||||||
|
val p = NextcloudProvider(account, chunkSize = 1L * 1024 * 1024) // 1 MB chunks
|
||||||
|
val dir = "SyncFlowChunk_${System.currentTimeMillis()}"
|
||||||
|
try {
|
||||||
|
p.createDirectory(dir).getOrThrow()
|
||||||
|
val payload = ByteArray(5 * 1024 * 1024 + 7).also { java.util.Random(7).nextBytes(it) } // ~5 MB -> 6 chunks
|
||||||
|
val up = p.uploadFile(ByteArrayInputStream(payload), "$dir/big.bin", payload.size.toLong())
|
||||||
|
assertTrue("chunked upload failed: ${up.exceptionOrNull()}", up.isSuccess)
|
||||||
|
assertEquals(payload.size.toLong(), p.listFiles(dir).getOrThrow().first { it.name == "big.bin" }.sizeBytes)
|
||||||
|
val out = ByteArrayOutputStream(); p.downloadFile("$dir/big.bin", out).getOrThrow()
|
||||||
|
assertArrayEquals("chunk-assembled content must equal the original bytes", payload, out.toByteArray())
|
||||||
|
} finally {
|
||||||
|
runCatching { p.deleteFile(dir) }
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,13 +2,111 @@ package com.syncflow.data.providers.nextcloud
|
|||||||
|
|
||||||
import com.syncflow.data.providers.webdav.WebDavProvider
|
import com.syncflow.data.providers.webdav.WebDavProvider
|
||||||
import com.syncflow.domain.model.CloudAccount
|
import com.syncflow.domain.model.CloudAccount
|
||||||
|
import com.syncflow.domain.model.RemoteFile
|
||||||
|
import kotlinx.coroutines.Dispatchers
|
||||||
|
import kotlinx.coroutines.withContext
|
||||||
|
import okhttp3.MediaType.Companion.toMediaType
|
||||||
|
import okhttp3.Request
|
||||||
|
import okhttp3.RequestBody
|
||||||
|
import okio.BufferedSink
|
||||||
|
import java.io.IOException
|
||||||
|
import java.io.InputStream
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Nextcloud WebDAV provider. Endpoint is /remote.php/dav/files/<username>/.
|
||||||
|
*
|
||||||
|
* Large files are uploaded via Nextcloud's chunked-upload API (/remote.php/dav/uploads/<user>/)
|
||||||
|
* so they bypass per-request size caps (Apache LimitRequestBody, PHP post_max_size, proxy body
|
||||||
|
* limits) that otherwise 413 a single multi-GB PUT. The assembly MOVE is the atomic commit, so
|
||||||
|
* the destination only appears once every chunk is in — no temp-file dance needed for this path.
|
||||||
|
*
|
||||||
|
* @param chunkSize bytes per chunk; files at or below this use the parent's single-PUT path.
|
||||||
|
*/
|
||||||
|
class NextcloudProvider(
|
||||||
|
account: CloudAccount,
|
||||||
|
private val chunkSize: Long = 100L * 1024 * 1024,
|
||||||
|
) : WebDavProvider(account) {
|
||||||
|
|
||||||
class NextcloudProvider(account: CloudAccount) : WebDavProvider(account) {
|
|
||||||
// Nextcloud WebDAV endpoint is /remote.php/dav/files/<username>/
|
|
||||||
override val baseUrl: String
|
override val baseUrl: String
|
||||||
get() {
|
get() {
|
||||||
val server = account.serverUrl?.trimEnd('/') ?: ""
|
val server = account.serverUrl?.trimEnd('/') ?: ""
|
||||||
val email = account.email ?: "user"
|
val email = account.email ?: "user"
|
||||||
return "$server/remote.php/dav/files/$email"
|
return "$server/remote.php/dav/files/$email"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private val uploadsBase: String
|
||||||
|
get() {
|
||||||
|
val server = account.serverUrl?.trimEnd('/') ?: ""
|
||||||
|
val email = account.email ?: "user"
|
||||||
|
return "$server/remote.php/dav/uploads/$email"
|
||||||
|
}
|
||||||
|
|
||||||
|
override suspend fun uploadFile(
|
||||||
|
localStream: InputStream,
|
||||||
|
remotePath: String,
|
||||||
|
sizeBytes: Long,
|
||||||
|
onProgress: (Long) -> Unit,
|
||||||
|
): Result<RemoteFile> {
|
||||||
|
if (sizeBytes <= chunkSize) {
|
||||||
|
return super.uploadFile(localStream, remotePath, sizeBytes, onProgress)
|
||||||
|
}
|
||||||
|
return runCatching {
|
||||||
|
withContext(Dispatchers.IO) {
|
||||||
|
val uploadId = "syncflow-${System.currentTimeMillis()}-${(0..999_999).random()}"
|
||||||
|
val dir = "$uploadsBase/$uploadId"
|
||||||
|
mkcol(dir)
|
||||||
|
try {
|
||||||
|
var index = 1
|
||||||
|
var sent = 0L
|
||||||
|
while (sent < sizeBytes) {
|
||||||
|
val len = minOf(chunkSize, sizeBytes - sent)
|
||||||
|
putChunk("$dir/%05d".format(index), localStream, len)
|
||||||
|
sent += len
|
||||||
|
index++
|
||||||
|
onProgress(sent)
|
||||||
|
}
|
||||||
|
// Assemble: MOVE the virtual .file onto the destination (atomic commit).
|
||||||
|
val move = Request.Builder().url("$dir/.file")
|
||||||
|
.method("MOVE", null)
|
||||||
|
.header("Destination", url(remotePath))
|
||||||
|
.header("Overwrite", "T")
|
||||||
|
.header("OC-Total-Length", sizeBytes.toString())
|
||||||
|
.build()
|
||||||
|
client.newCall(move).execute().use { resp ->
|
||||||
|
if (!resp.isSuccessful) throw IOException("Chunk assembly MOVE HTTP ${resp.code}")
|
||||||
|
}
|
||||||
|
} catch (e: Throwable) {
|
||||||
|
runCatching { client.newCall(Request.Builder().url(dir).delete().build()).execute().close() }
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
getFileMetadata(remotePath).getOrThrow()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun mkcol(url: String) {
|
||||||
|
client.newCall(Request.Builder().url(url).method("MKCOL", null).build()).execute().use {
|
||||||
|
if (!it.isSuccessful && it.code != 405) throw IOException("MKCOL upload session HTTP ${it.code}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun putChunk(url: String, stream: InputStream, len: Long) {
|
||||||
|
val body = object : RequestBody() {
|
||||||
|
override fun contentType() = "application/octet-stream".toMediaType()
|
||||||
|
override fun contentLength() = len
|
||||||
|
override fun writeTo(sink: BufferedSink) {
|
||||||
|
var remaining = len
|
||||||
|
val buf = ByteArray(64 * 1024)
|
||||||
|
while (remaining > 0) {
|
||||||
|
val n = stream.read(buf, 0, minOf(buf.size.toLong(), remaining).toInt())
|
||||||
|
if (n < 0) break
|
||||||
|
sink.write(buf, 0, n)
|
||||||
|
remaining -= n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
client.newCall(Request.Builder().url(url).put(body).build()).execute().use {
|
||||||
|
if (!it.isSuccessful) throw IOException("Chunk PUT HTTP ${it.code}")
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-2
@@ -1,2 +1,2 @@
|
|||||||
VERSION_NAME=1.0.64
|
VERSION_NAME=1.0.65
|
||||||
VERSION_CODE=65
|
VERSION_CODE=66
|
||||||
|
|||||||
Reference in New Issue
Block a user