bitforge/cf-uploader
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Try out chunking

Browse Source
This commit is contained in:
pratik
2025-10-21 15:19:49 -05:00
parent 9ad9e6b7b8
commit fdcc92eeb6
9 changed files with 849 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/cfdeployer/CfDeployerApplication.java
View File

@@ -2,8 +2,10 @@ package com.cfdeployer;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.scheduling.annotation.EnableScheduling;
@SpringBootApplication
@EnableScheduling
public class CfDeployerApplication {
public static void main(String[] args) {

124
src/main/java/com/cfdeployer/controller/CfDeployController.java
View File

@@ -2,7 +2,10 @@ package com.cfdeployer.controller;
import com.cfdeployer.model.CfDeployRequest;
import com.cfdeployer.model.CfDeployResponse;
import com.cfdeployer.model.ChunkUploadRequest;
import com.cfdeployer.model.ChunkUploadResponse;
import com.cfdeployer.service.CfCliService;
import com.cfdeployer.service.ChunkedUploadService;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
@@ -22,6 +25,7 @@ public class CfDeployController {
private static final Logger log = LoggerFactory.getLogger(CfDeployController.class);
private final CfCliService cfCliService;
private final ChunkedUploadService chunkedUploadService;
private final ObjectMapper objectMapper;
@PostMapping(value = "/deploy", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
@@ -118,6 +122,126 @@ public class CfDeployController {
}
}
// Chunked upload endpoints
@PostMapping("/upload/init")
public ResponseEntity<ChunkUploadResponse> initUpload(@RequestBody String requestJson) {
try {
log.info("Initializing chunked upload session");
// Validate the request JSON
CfDeployRequest request = objectMapper.readValue(requestJson, CfDeployRequest.class);
log.info("Creating upload session for app: {}", request.getAppName());
String sessionId = chunkedUploadService.createUploadSession(requestJson);
return ResponseEntity.ok(ChunkUploadResponse.builder()
.success(true)
.uploadSessionId(sessionId)
.message("Upload session created successfully")
.build());
} catch (Exception e) {
log.error("Error initializing upload session", e);
return ResponseEntity.status(HttpStatus.BAD_REQUEST)
.body(ChunkUploadResponse.failure("Failed to initialize upload: " + e.getMessage()));
}
}
@PostMapping("/upload/chunk")
public ResponseEntity<ChunkUploadResponse> uploadChunk(
@RequestParam("uploadSessionId") String uploadSessionId,
@RequestParam("fileType") String fileType,
@RequestParam("chunkIndex") Integer chunkIndex,
@RequestParam("totalChunks") Integer totalChunks,
@RequestParam(value = "fileName", required = false) String fileName,
@RequestPart("chunk") MultipartFile chunk) {
try {
log.debug("Receiving chunk {}/{} for session: {}, fileType: {}",
chunkIndex + 1, totalChunks, uploadSessionId, fileType);
// Validate file type
if (!fileType.equals("jarFile") && !fileType.equals("manifest")) {
throw new IllegalArgumentException("Invalid file type. Must be 'jarFile' or 'manifest'");
}
chunkedUploadService.uploadChunk(uploadSessionId, fileType, fileName,
chunkIndex, totalChunks, chunk);
var session = chunkedUploadService.getSession(uploadSessionId);
var fileState = session.getFileStates().get(fileType);
return ResponseEntity.ok(ChunkUploadResponse.success(
uploadSessionId, fileType, chunkIndex, totalChunks,
fileState.getReceivedChunkCount()));
} catch (Exception e) {
log.error("Error uploading chunk", e);
return ResponseEntity.status(HttpStatus.BAD_REQUEST)
.body(ChunkUploadResponse.failure("Failed to upload chunk: " + e.getMessage()));
}
}
@PostMapping("/upload/finalize")
public ResponseEntity<CfDeployResponse> finalizeUpload(@RequestParam("uploadSessionId") String uploadSessionId) {
try {
log.info("Finalizing upload for session: {}", uploadSessionId);
if (!chunkedUploadService.isSessionReady(uploadSessionId)) {
return ResponseEntity.status(HttpStatus.BAD_REQUEST)
.body(CfDeployResponse.failure("Upload incomplete. Not all file chunks received.", null));
}
var session = chunkedUploadService.getSession(uploadSessionId);
if (session == null) {
return ResponseEntity.status(HttpStatus.NOT_FOUND)
.body(CfDeployResponse.failure("Upload session not found or expired", null));
}
CfDeployRequest request = objectMapper.readValue(session.getRequestJson(), CfDeployRequest.class);
log.info("Starting deployment for app: {} from session: {}",
request.getAppName(), uploadSessionId);
// Get file paths from session
var jarState = session.getFileStates().get("jarFile");
var manifestState = session.getFileStates().get("manifest");
CfDeployResponse response = cfCliService.deployApplicationFromPaths(
request,
jarState.getTargetPath(),
manifestState.getTargetPath());
// Clean up session after deployment
chunkedUploadService.deleteSession(uploadSessionId);
if (Boolean.TRUE.equals(response.getSuccess())) {
return ResponseEntity.ok(response);
} else {
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body(response);
}
} catch (Exception e) {
log.error("Error finalizing upload", e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
.body(CfDeployResponse.failure(
"Failed to finalize deployment: " + e.getMessage(),
e.toString()));
}
}
@GetMapping("/upload/status/{uploadSessionId}")
public ResponseEntity<?> getUploadStatus(@PathVariable String uploadSessionId) {
try {
var session = chunkedUploadService.getSession(uploadSessionId);
if (session == null) {
return ResponseEntity.status(HttpStatus.NOT_FOUND)
.body("Upload session not found or expired");
}
return ResponseEntity.ok(session.getFileStates());
} catch (Exception e) {
log.error("Error getting upload status", e);
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
.body("Failed to get upload status: " + e.getMessage());
}
}
private void validateFiles(MultipartFile jarFile, MultipartFile manifest) {
if (jarFile.isEmpty()) {
throw new IllegalArgumentException("JAR file is empty");

26
src/main/java/com/cfdeployer/model/ChunkUploadRequest.java Normal file
View File

@@ -0,0 +1,26 @@
package com.cfdeployer.model;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
@Data
public class ChunkUploadRequest {
@NotBlank(message = "Upload session ID is required")
private String uploadSessionId;
@NotBlank(message = "File type is required (jarFile or manifest)")
private String fileType; // "jarFile" or "manifest"
@NotNull(message = "Chunk index is required")
@Min(value = 0, message = "Chunk index must be non-negative")
private Integer chunkIndex;
@NotNull(message = "Total chunks is required")
@Min(value = 1, message = "Total chunks must be at least 1")
private Integer totalChunks;
private String fileName;
}

41
src/main/java/com/cfdeployer/model/ChunkUploadResponse.java Normal file
View File

@@ -0,0 +1,41 @@
package com.cfdeployer.model;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class ChunkUploadResponse {
private Boolean success;
private String uploadSessionId;
private String fileType;
private Integer chunkIndex;
private Integer totalChunks;
private Integer receivedChunks;
private String message;
public static ChunkUploadResponse success(String uploadSessionId, String fileType,
Integer chunkIndex, Integer totalChunks, Integer receivedChunks) {
return ChunkUploadResponse.builder()
.success(true)
.uploadSessionId(uploadSessionId)
.fileType(fileType)
.chunkIndex(chunkIndex)
.totalChunks(totalChunks)
.receivedChunks(receivedChunks)
.message("Chunk uploaded successfully")
.build();
}
public static ChunkUploadResponse failure(String message) {
return ChunkUploadResponse.builder()
.success(false)
.message(message)
.build();
}
}

61
src/main/java/com/cfdeployer/model/UploadSession.java Normal file
View File

@@ -0,0 +1,61 @@
package com.cfdeployer.model;
import lombok.Data;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@Data
public class UploadSession {
private String sessionId;
private String requestJson;
private Path workingDirectory;
private LocalDateTime createdAt;
private LocalDateTime lastAccessedAt;
// File type -> chunk tracking
private Map<String, FileUploadState> fileStates;
public UploadSession(String sessionId, String requestJson, Path workingDirectory) {
this.sessionId = sessionId;
this.requestJson = requestJson;
this.workingDirectory = workingDirectory;
this.createdAt = LocalDateTime.now();
this.lastAccessedAt = LocalDateTime.now();
this.fileStates = new ConcurrentHashMap<>();
}
public void updateLastAccessed() {
this.lastAccessedAt = LocalDateTime.now();
}
@Data
public static class FileUploadState {
private String fileName;
private int totalChunks;
private Map<Integer, Boolean> receivedChunks;
private Path targetPath;
public FileUploadState(String fileName, int totalChunks, Path targetPath) {
this.fileName = fileName;
this.totalChunks = totalChunks;
this.targetPath = targetPath;
this.receivedChunks = new ConcurrentHashMap<>();
}
public void markChunkReceived(int chunkIndex) {
receivedChunks.put(chunkIndex, true);
}
public boolean isComplete() {
return receivedChunks.size() == totalChunks;
}
public int getReceivedChunkCount() {
return receivedChunks.size();
}
}
}

38
src/main/java/com/cfdeployer/service/CfCliService.java
View File

@@ -84,6 +84,41 @@ public class CfCliService {
}
}
public CfDeployResponse deployApplicationFromPaths(CfDeployRequest request, Path jarPath, Path manifestPath) {
try {
log.info("=== Starting deployment from paths for app: {} ===", request.getAppName());
log.info("Target: {}/{}/{}", request.getApiEndpoint(), request.getOrganization(), request.getSpace());
log.info("JAR path: {}", jarPath);
log.info("Manifest path: {}", manifestPath);
// Validate files exist
if (!Files.exists(jarPath)) {
throw new IOException("JAR file not found at: " + jarPath);
}
if (!Files.exists(manifestPath)) {
throw new IOException("Manifest file not found at: " + manifestPath);
}
log.info("JAR file size: {} bytes", Files.size(jarPath));
log.info("Manifest file size: {} bytes", Files.size(manifestPath));
StringBuilder output = new StringBuilder();
login(request, output);
pushApplication(request, manifestPath.getParent(), jarPath, output);
logout(output);
log.info("=== Deployment completed successfully for app: {} ===", request.getAppName());
return CfDeployResponse.success(output.toString());
} catch (Exception e) {
log.error("=== Deployment failed for app: {} ===", request.getAppName());
log.error("Error type: {}", e.getClass().getName());
log.error("Error message: {}", e.getMessage(), e);
return CfDeployResponse.failure(e.getMessage(), e.toString());
}
}
private void login(CfDeployRequest request, StringBuilder output) throws Exception {
log.info("Logging into Cloud Foundry at: {}", request.getApiEndpoint());
@@ -224,7 +259,8 @@ public class CfCliService {
log.info("Created temp file: {}", tempFile.getAbsolutePath());
// Copy from direct file path to temp file
long bytesCopied = Files.copy(directPath, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
Files.copy(directPath, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
long bytesCopied = Files.size(tempFile.toPath());
log.info("Copied CF CLI to temp file: {} ({} bytes)", tempFile.getAbsolutePath(), bytesCopied);
if (bytesCopied == 0) {

161
src/main/java/com/cfdeployer/service/ChunkedUploadService.java Normal file
View File

@@ -0,0 +1,161 @@
package com.cfdeployer.service;
import com.cfdeployer.model.UploadSession;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.Comparator;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
@Service
@Slf4j
public class ChunkedUploadService {
@Value("${cf.upload.session.timeout-minutes:30}")
private int sessionTimeoutMinutes;
private final Map<String, UploadSession> activeSessions = new ConcurrentHashMap<>();
public String createUploadSession(String requestJson) throws IOException {
String sessionId = UUID.randomUUID().toString();
Path workingDir = Files.createTempDirectory("cf-upload-" + sessionId);
UploadSession session = new UploadSession(sessionId, requestJson, workingDir);
activeSessions.put(sessionId, session);
log.info("Created upload session: {} at {}", sessionId, workingDir);
return sessionId;
}
public synchronized void uploadChunk(String sessionId, String fileType, String fileName,
int chunkIndex, int totalChunks, MultipartFile chunk) throws IOException {
UploadSession session = activeSessions.get(sessionId);
if (session == null) {
throw new IllegalArgumentException("Upload session not found or expired: " + sessionId);
}
session.updateLastAccessed();
// Get or create file upload state
UploadSession.FileUploadState fileState = session.getFileStates()
.computeIfAbsent(fileType, k -> {
String targetFileName = fileType.equals("manifest") ? "manifest.yml" : fileName;
Path targetPath = session.getWorkingDirectory().resolve(targetFileName);
return new UploadSession.FileUploadState(fileName, totalChunks, targetPath);
});
// Validate total chunks consistency
if (fileState.getTotalChunks() != totalChunks) {
throw new IllegalArgumentException(
String.format("Total chunks mismatch for %s: expected %d, got %d",
fileType, fileState.getTotalChunks(), totalChunks));
}
// Write chunk to file
Path targetPath = fileState.getTargetPath();
long offset = (long) chunkIndex * getChunkSize();
try (RandomAccessFile raf = new RandomAccessFile(targetPath.toFile(), "rw")) {
raf.seek(offset);
byte[] data = chunk.getBytes();
raf.write(data);
log.debug("Wrote chunk {} ({} bytes) to {} at offset {}",
chunkIndex, data.length, targetPath.getFileName(), offset);
}
fileState.markChunkReceived(chunkIndex);
log.info("Session {}: Received chunk {}/{} for {} ({} bytes)",
sessionId, chunkIndex + 1, totalChunks, fileType, chunk.getSize());
if (fileState.isComplete()) {
log.info("Session {}: File {} upload completed ({} chunks)",
sessionId, fileType, totalChunks);
}
}
public UploadSession getSession(String sessionId) {
UploadSession session = activeSessions.get(sessionId);
if (session != null) {
session.updateLastAccessed();
}
return session;
}
public boolean isSessionReady(String sessionId) {
UploadSession session = activeSessions.get(sessionId);
if (session == null) {
return false;
}
// Check if both jarFile and manifest are complete
UploadSession.FileUploadState jarState = session.getFileStates().get("jarFile");
UploadSession.FileUploadState manifestState = session.getFileStates().get("manifest");
return jarState != null && jarState.isComplete() &&
manifestState != null && manifestState.isComplete();
}
public void deleteSession(String sessionId) {
UploadSession session = activeSessions.remove(sessionId);
if (session != null) {
cleanupSessionDirectory(session);
log.info("Deleted upload session: {}", sessionId);
}
}
@Scheduled(fixedRate = 300000) // Run every 5 minutes
public void cleanupExpiredSessions() {
LocalDateTime expirationTime = LocalDateTime.now().minusMinutes(sessionTimeoutMinutes);
int cleanedCount = 0;
for (Map.Entry<String, UploadSession> entry : activeSessions.entrySet()) {
if (entry.getValue().getLastAccessedAt().isBefore(expirationTime)) {
deleteSession(entry.getKey());
cleanedCount++;
}
}
if (cleanedCount > 0) {
log.info("Cleaned up {} expired upload sessions", cleanedCount);
}
}
private void cleanupSessionDirectory(UploadSession session) {
try {
Path workingDir = session.getWorkingDirectory();
if (Files.exists(workingDir)) {
Files.walk(workingDir)
.sorted(Comparator.reverseOrder())
.forEach(path -> {
try {
Files.delete(path);
} catch (IOException e) {
log.warn("Failed to delete file: {}", path, e);
}
});
log.debug("Cleaned up session directory: {}", workingDir);
}
} catch (IOException e) {
log.warn("Failed to clean up session directory for session: {}", session.getSessionId(), e);
}
}
private int getChunkSize() {
// Default chunk size - should match client-side
return 5 * 1024 * 1024; // 5MB
}
public int getActiveSessionCount() {
return activeSessions.size();
}
}

8
src/main/resources/application.properties
View File

@@ -4,7 +4,7 @@ server.port=8080
# Application Name
spring.application.name=cf-deployer
# Multipart Configuration
# Multipart Configuration - for traditional single upload endpoint
spring.servlet.multipart.max-file-size=500MB
spring.servlet.multipart.max-request-size=500MB
spring.servlet.multipart.enabled=true
@@ -13,6 +13,12 @@ spring.servlet.multipart.enabled=true
cf.cli.timeout=600
cf.cli.path=
# Chunked Upload Configuration
# Recommended chunk size: 5MB (client-side should match this)
cf.upload.chunk.size=5242880
# Session timeout in minutes (default: 30 minutes)
cf.upload.session.timeout-minutes=30
# Logging Configuration
logging.level.root=INFO
logging.level.com.cfdeployer=DEBUG