What is multipart? ?

What's special about multipart is that we can upload very large files, up to Terabyte size.

The functioning of multipart is somewhat particular. Typically, when uploading a standard file, the entire file is sent as a single object. With multipart, the file is divided into several parts, which are then reassembled together.

Create file multipart

1. The file is divided into several parts, called chunks. In this case, each segment is 5 MB in size.

const chunkSize = 1024 * 1024 * 5; // 5 MB chunk size
const totalChunks = Math.ceil(file.size / chunkSize);

const createFileChunks = (file, chunkSize) => {
    const chunks: Blob[] = [];
    let offset = 0;

    while (offset < file.size) {
        chunks.push(file.slice(offset, offset + chunkSize));
        offset += chunkSize;
    }

    return chunks;
}

2. Multipart is created by sending the filename to be uploaded along with the total number of chunks.

const initResponst = async () => {
    const response = await fetch("/folders/upload/init", {
        method: "POST",
        headers: {
            "Content-Type": "application/json",
        },
        body: JSON.stringify({
            filename: file.name,
            total_chunks: totalChunks,
        }),
    });

    if (!response.ok) {
        throw new Error(`HTTP error! status: ${response.status}`);
    }

    return await response.json();
}

3. The creation of the multipart upload is sent to the S3 server, ensuring that the upload ID is returned to continue the process.

val uploads = mutableMapOf<String, MutableList<CompletedPart>>()

suspend fun initiateMultipartUpload(client: S3Client, key: String): String? {
    val multipartRes = client.createMultipartUpload {
        checksumAlgorithm = ChecksumAlgorithm.Sha256
        bucket = S3Config.bucketName
        this.key = key
    }

    uploads[multipartRes.uploadId!!] = mutableListOf()

    return multipartRes.uploadId
}

Upload all the parts of the file as chunks.

1. The formData is created and sent to the API.

for (let index = 0; index < chunks.length; index++) {
    const chunk = chunks[index];
    const formData = new FormData();
    formData.append('uploadId', uploadId.toString());
    formData.append('chunkNumber', (index + 1).toString());
    formData.append('totalChunks', totalChunks.toString());
    formData.append('file', chunk, file.name);

    await fetch("/folders/upload", {
        method: "POST",
        headers: {
            "Content-Type": "multipart/form-data",
        },
        body: formData,
    });
}

2. Retrieve the formData to then upload the sent chunk.

Thanks to the receiveMultipart() method from Ktor, you can easily retrieve the formData pand send it to the S3 server.

post {
    val multipart = call.receiveMultipart()
    var uploadId: String? = null
    var chunkNumber: Int? = null
    var totalChunks: Int? = null
    var originalFileName: String? = null
    var fileBytes: ByteArray? = null

    multipart.forEachPart { part ->
        when (part) {
            is PartData.FormItem -> {
                when (part.name) {
                    "uploadId" -> uploadId = part.value
                    "chunkNumber" -> chunkNumber = part.value.toIntOrNull()
                    "totalChunks" -> totalChunks = part.value.toIntOrNull()
                }
            }

            is PartData.FileItem -> {
                originalFileName = part.originalFileName
                fileBytes = part.streamProvider().readBytes()
            }

            else -> {}
        }
        part.dispose()
    }

    if (uploadId != null && chunkNumber != null && totalChunks != null && originalFileName != null && fileBytes != null) {
            try {
                S3Config.makeClient()?.let {
                    S3SystemService.uploadMultipart(it, originalFileName, uploadId, chunkNumber!!, fileBytes, totalChunks!!)
                }
                call.respond(HttpStatusCode.OK, "Chunk $chunkNumber uploaded successfully")
            } catch (e: S3Exception) {
                call.respond(HttpStatusCode.BadRequest, "Error uploading chunk ${e.message}")
            }
    } else {
        call.respond(HttpStatusCode.BadRequest, "Invalid upload data")
    }
}

Then, the chunk is uploaded to the S3 server and stored for reassembling the file.

suspend fun uploadMultipart(client: S3Client, key: String, uploadId: String?, chunkNumber: Int, fileBytes: ByteArray?, totalChunks: Int): String? {
    try {
        val part = client.uploadPart(UploadPartRequest {
            bucket = S3Config.bucketName
            this.key = key
            this.uploadId = uploadId
            partNumber = chunkNumber
            body = ByteStream.fromBytes(fileBytes!!)
        }).let {
            CompletedPart {
                checksumSha256 = it.checksumSha256
                partNumber = chunkNumber
                eTag = it.eTag
            }
        }

        uploads[uploadId!!]?.add(part)

    } catch (e: S3Exception) {
       println("Error uploading file: ${e.message}")
    }

    return uploadId
}

Reassembling the file

1. Indicate the end of the file upload to reassemble it.

await fetch("/folders/upload/complete", {
    method: "POST",
    headers: {
        "Content-Type": "application/json",
    },
    body: JSON.stringify({
        upload_id: uploadId.toString(),
        filename: file.name,
    }),
});

2. The file is reassembled on the S3 server.

As we can see, the uploadId tracked the entire multipart upload process. Using this identifier and the filename, S3 locates the chunks and reassembles the file

suspend fun completeMultipartUpload(client: S3Client, remotePath: String, uplId: String?) {
    client.completeMultipartUpload(CompleteMultipartUploadRequest {
        bucket = S3Config.bucketName
        this.key = remotePath
        this.uploadId = uplId
        multipartUpload = CompletedMultipartUpload {
            parts = uploads[uplId!!]
            bucket = S3Config.bucketName
            key = remotePath
            uploadId = uplId
        }
    }).also { uploads.remove(uplId) }
}