Dynamic size

This commit is contained in:
pratik
2025-10-21 18:48:35 -05:00
parent 6e3684b580
commit b81ab51100
3 changed files with 40 additions and 25 deletions

View File

@@ -61,13 +61,21 @@ public class ChunkedUploadService {
fileType, fileState.getTotalChunks(), totalChunks));
}
// Write chunk to file using streaming to avoid loading entire chunk into memory
// Write chunk to file using sequential append mode
// This supports variable chunk sizes - chunks MUST be uploaded in order (0, 1, 2, ...)
Path targetPath = fileState.getTargetPath();
long offset = (long) chunkIndex * getChunkSize();
try (RandomAccessFile raf = new RandomAccessFile(targetPath.toFile(), "rw");
var inputStream = chunk.getInputStream()) {
raf.seek(offset);
// Verify chunks are uploaded in order
if (chunkIndex != fileState.getReceivedChunkCount()) {
throw new IllegalArgumentException(
String.format("Chunks must be uploaded in order. Expected chunk %d but received %d",
fileState.getReceivedChunkCount(), chunkIndex));
}
try (var inputStream = chunk.getInputStream();
var outputStream = Files.newOutputStream(targetPath,
java.nio.file.StandardOpenOption.CREATE,
java.nio.file.StandardOpenOption.APPEND)) {
// Stream chunk data in smaller buffers to reduce memory pressure
byte[] buffer = new byte[8192]; // 8KB buffer
@@ -75,12 +83,12 @@ public class ChunkedUploadService {
long totalWritten = 0;
while ((bytesRead = inputStream.read(buffer)) != -1) {
raf.write(buffer, 0, bytesRead);
outputStream.write(buffer, 0, bytesRead);
totalWritten += bytesRead;
}
log.debug("Wrote chunk {} ({} bytes) to {} at offset {}",
chunkIndex, totalWritten, targetPath.getFileName(), offset);
log.debug("Appended chunk {} ({} bytes) to {}",
chunkIndex, totalWritten, targetPath.getFileName());
}
fileState.markChunkReceived(chunkIndex);
@@ -160,11 +168,6 @@ public class ChunkedUploadService {
}
}
private int getChunkSize() {
// Default chunk size - should match client-side
return 5 * 1024 * 1024; // 5MB
}
public int getActiveSessionCount() {
return activeSessions.size();
}

View File

@@ -16,10 +16,10 @@ cf.cli.timeout=600
cf.cli.path=
# Chunked Upload Configuration
# Reduced chunk size to 2MB to avoid memory issues on low-memory Tanzu instances
cf.upload.chunk.size=2097152
# Session timeout in minutes (default: 30 minutes)
cf.upload.session.timeout-minutes=30
# Note: Chunk size is controlled by the client, not the server.
# Server accepts any chunk size and appends chunks sequentially.
# Logging Configuration
logging.level.root=INFO