Add offline detection, chunked uploads, and security tests (#9, #10, #12, #15)

- Add offline detection with navigator.onLine and auto-pause/resume - Implement chunked upload for files >100MB with localStorage persistence - Add download by artifact ID input field in PackagePage - Add 10 security tests for path traversal and malformed requests - Fix test setup globalThis reference
2026-01-07 14:24:44 -06:00
parent b9b4334393
commit 0302e5b21a
6 changed files with 689 additions and 37 deletions
--- a/frontend/src/components/DragDropUpload.tsx
+++ b/frontend/src/components/DragDropUpload.tsx
@@ -1,8 +1,63 @@
 import { useState, useRef, useCallback, useEffect } from 'react';
 import './DragDropUpload.css';

+const CHUNK_SIZE = 10 * 1024 * 1024;
+const CHUNKED_UPLOAD_THRESHOLD = 100 * 1024 * 1024;
+const UPLOAD_STATE_PREFIX = 'orchard_upload_';
+
+interface StoredUploadState {
+  uploadId: string;
+  fileHash: string;
+  filename: string;
+  fileSize: number;
+  completedParts: number[];
+  project: string;
+  package: string;
+  tag?: string;
+  createdAt: number;
+}
+
+function getUploadStateKey(project: string, pkg: string, fileHash: string): string {
+  return `${UPLOAD_STATE_PREFIX}${project}_${pkg}_${fileHash}`;
+}
+
+function saveUploadState(state: StoredUploadState): void {
+  try {
+    const key = getUploadStateKey(state.project, state.package, state.fileHash);
+    localStorage.setItem(key, JSON.stringify(state));
+  } catch {
+    // localStorage might be full or unavailable
+  }
+}
+
+function loadUploadState(project: string, pkg: string, fileHash: string): StoredUploadState | null {
+  try {
+    const key = getUploadStateKey(project, pkg, fileHash);
+    const stored = localStorage.getItem(key);
+    if (!stored) return null;
+    const state = JSON.parse(stored) as StoredUploadState;
+    const oneDay = 24 * 60 * 60 * 1000;
+    if (Date.now() - state.createdAt > oneDay) {
+      localStorage.removeItem(key);
+      return null;
+    }
+    return state;
+  } catch {
+    return null;
+  }
+}
+
+function clearUploadState(project: string, pkg: string, fileHash: string): void {
+  try {
+    const key = getUploadStateKey(project, pkg, fileHash);
+    localStorage.removeItem(key);
+  } catch {
+    // ignore
+  }
+}
+
 // Types
-export type UploadStatus = 'pending' | 'uploading' | 'complete' | 'failed' | 'validating';
+export type UploadStatus = 'pending' | 'uploading' | 'complete' | 'failed' | 'validating' | 'paused';

 export interface UploadItem {
  id: string;
@@ -61,7 +116,6 @@ function formatTimeRemaining(seconds: number): string {
 }

 function getFileExtension(filename: string): string {
-  // Handle compound extensions like .tar.gz
  const parts = filename.toLowerCase().split('.');
  if (parts.length >= 3 && parts[parts.length - 2] === 'tar') {
    return `.${parts.slice(-2).join('.')}`;
@@ -69,6 +123,13 @@ function getFileExtension(filename: string): string {
  return parts.length > 1 ? `.${parts[parts.length - 1]}` : '';
 }

+async function computeSHA256(file: File): Promise<string> {
+  const buffer = await file.arrayBuffer();
+  const hashBuffer = await crypto.subtle.digest('SHA-256', buffer);
+  const hashArray = Array.from(new Uint8Array(hashBuffer));
+  return hashArray.map(b => b.toString(16).padStart(2, '0')).join('');
+}
+
 // Icons
 function UploadIcon() {
  return (
@@ -125,6 +186,38 @@ function FileIcon() {
  );
 }

+function PauseIcon() {
+  return (
+    <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2">
+      <rect x="6" y="4" width="4" height="16" />
+      <rect x="14" y="4" width="4" height="16" />
+    </svg>
+  );
+}
+
+function WifiOffIcon() {
+  return (
+    <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2">
+      <line x1="1" y1="1" x2="23" y2="23" />
+      <path d="M16.72 11.06A10.94 10.94 0 0 1 19 12.55" />
+      <path d="M5 12.55a10.94 10.94 0 0 1 5.17-2.39" />
+      <path d="M10.71 5.05A16 16 0 0 1 22.58 9" />
+      <path d="M1.42 9a15.91 15.91 0 0 1 4.7-2.88" />
+      <path d="M8.53 16.11a6 6 0 0 1 6.95 0" />
+      <line x1="12" y1="20" x2="12.01" y2="20" />
+    </svg>
+  );
+}
+
+function SpinnerIcon() {
+  return (
+    <svg width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" className="spinner-icon">
+      <circle cx="12" cy="12" r="10" strokeOpacity="0.25" />
+      <path d="M12 2a10 10 0 0 1 10 10" />
+    </svg>
+  );
+}
+
 export function DragDropUpload({
  projectName,
  packageName,
@@ -140,9 +233,52 @@ export function DragDropUpload({
 }: DragDropUploadProps) {
  const [isDragOver, setIsDragOver] = useState(false);
  const [uploadQueue, setUploadQueue] = useState<UploadItem[]>([]);
+  const [isOnline, setIsOnline] = useState(navigator.onLine);
  const fileInputRef = useRef<HTMLInputElement>(null);
  const dragCounterRef = useRef(0);
  const activeUploadsRef = useRef(0);
+  const xhrMapRef = useRef<Map<string, XMLHttpRequest>>(new Map());
+
+  // Online/Offline detection
+  useEffect(() => {
+    const handleOnline = () => {
+      setIsOnline(true);
+      // Resume paused uploads
+      setUploadQueue(prev => prev.map(item => 
+        item.status === 'paused' 
+          ? { ...item, status: 'pending' as UploadStatus, error: undefined }
+          : item
+      ));
+    };
+
+    const handleOffline = () => {
+      setIsOnline(false);
+      // Pause uploading items and cancel their XHR requests
+      setUploadQueue(prev => prev.map(item => {
+        if (item.status === 'uploading') {
+          // Abort the XHR request
+          const xhr = xhrMapRef.current.get(item.id);
+          if (xhr) {
+            xhr.abort();
+            xhrMapRef.current.delete(item.id);
+          }
+          return { ...item, status: 'paused' as UploadStatus, error: 'Network offline - will resume when connection is restored', progress: 0 };
+        }
+        if (item.status === 'pending') {
+          return { ...item, status: 'paused' as UploadStatus, error: 'Network offline - waiting for connection' };
+        }
+        return item;
+      }));
+    };
+
+    window.addEventListener('online', handleOnline);
+    window.addEventListener('offline', handleOffline);
+
+    return () => {
+      window.removeEventListener('online', handleOnline);
+      window.removeEventListener('offline', handleOffline);
+    };
+  }, []);

  // Validate a single file
  const validateFile = useCallback((file: File): string | null => {
@@ -185,10 +321,163 @@ export function DragDropUpload({
    setUploadQueue(prev => [...prev, ...newItems]);
  }, [validateFile]);

-  // Upload a single file with progress tracking
-  const uploadFile = useCallback((item: UploadItem): Promise<UploadResult> => {
+  const uploadFileChunked = useCallback(async (item: UploadItem): Promise<UploadResult> => {
+    setUploadQueue(prev => prev.map(u => 
+      u.id === item.id 
+        ? { ...u, status: 'validating' as UploadStatus, startTime: Date.now() }
+        : u
+    ));
+
+    const fileHash = await computeSHA256(item.file);
+    
+    const storedState = loadUploadState(projectName, packageName, fileHash);
+    let uploadId: string;
+    let completedParts: number[] = [];
+
+    if (storedState && storedState.fileSize === item.file.size && storedState.filename === item.file.name) {
+      try {
+        const statusResponse = await fetch(
+          `/api/v1/project/${projectName}/${packageName}/upload/${storedState.uploadId}/status`
+        );
+        if (statusResponse.ok) {
+          const statusData = await statusResponse.json();
+          uploadId = storedState.uploadId;
+          completedParts = statusData.uploaded_parts || [];
+        } else {
+          throw new Error('Stored upload no longer valid');
+        }
+      } catch {
+        clearUploadState(projectName, packageName, fileHash);
+        uploadId = await initNewUpload();
+      }
+    } else {
+      uploadId = await initNewUpload();
+    }
+
+    async function initNewUpload(): Promise<string> {
+      const initResponse = await fetch(
+        `/api/v1/project/${projectName}/${packageName}/upload/init`,
+        {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({
+            expected_hash: fileHash,
+            filename: item.file.name,
+            size: item.file.size,
+            tag: tag || undefined,
+          }),
+        }
+      );
+
+      if (!initResponse.ok) {
+        const error = await initResponse.json().catch(() => ({}));
+        throw new Error(error.detail || `Init failed: ${initResponse.status}`);
+      }
+
+      const initData = await initResponse.json();
+
+      if (initData.already_exists) {
+        throw { deduplicated: true, artifact_id: initData.artifact_id };
+      }
+
+      saveUploadState({
+        uploadId: initData.upload_id,
+        fileHash,
+        filename: item.file.name,
+        fileSize: item.file.size,
+        completedParts: [],
+        project: projectName,
+        package: packageName,
+        tag: tag || undefined,
+        createdAt: Date.now(),
+      });
+
+      return initData.upload_id;
+    }
+
+    const totalChunks = Math.ceil(item.file.size / CHUNK_SIZE);
+    let uploadedBytes = completedParts.length * CHUNK_SIZE;
+    if (uploadedBytes > item.file.size) uploadedBytes = item.file.size - (item.file.size % CHUNK_SIZE);
+    const startTime = Date.now();
+
+    for (let partNumber = 1; partNumber <= totalChunks; partNumber++) {
+      if (completedParts.includes(partNumber)) {
+        continue;
+      }
+
+      if (!isOnline) {
+        throw new Error('Network offline');
+      }
+
+      const start = (partNumber - 1) * CHUNK_SIZE;
+      const end = Math.min(start + CHUNK_SIZE, item.file.size);
+      const chunk = item.file.slice(start, end);
+
+      const partResponse = await fetch(
+        `/api/v1/project/${projectName}/${packageName}/upload/${uploadId}/part/${partNumber}`,
+        {
+          method: 'PUT',
+          body: chunk,
+        }
+      );
+
+      if (!partResponse.ok) {
+        throw new Error(`Part ${partNumber} upload failed: ${partResponse.status}`);
+      }
+
+      completedParts.push(partNumber);
+      saveUploadState({
+        uploadId,
+        fileHash,
+        filename: item.file.name,
+        fileSize: item.file.size,
+        completedParts,
+        project: projectName,
+        package: packageName,
+        tag: tag || undefined,
+        createdAt: Date.now(),
+      });
+
+      uploadedBytes += chunk.size;
+      const elapsed = (Date.now() - startTime) / 1000;
+      const speed = elapsed > 0 ? uploadedBytes / elapsed : 0;
+      const progress = Math.round((uploadedBytes / item.file.size) * 100);
+
+      setUploadQueue(prev => prev.map(u => 
+        u.id === item.id 
+          ? { ...u, progress, speed, status: 'uploading' as UploadStatus }
+          : u
+      ));
+    }
+
+    const completeResponse = await fetch(
+      `/api/v1/project/${projectName}/${packageName}/upload/${uploadId}/complete`,
+      {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ tag: tag || undefined }),
+      }
+    );
+
+    if (!completeResponse.ok) {
+      throw new Error(`Complete failed: ${completeResponse.status}`);
+    }
+
+    clearUploadState(projectName, packageName, fileHash);
+
+    const completeData = await completeResponse.json();
+    return {
+      artifact_id: completeData.artifact_id,
+      size: completeData.size,
+      deduplicated: false,
+    };
+  }, [projectName, packageName, tag, isOnline]);
+
+  const uploadFileSimple = useCallback((item: UploadItem): Promise<UploadResult> => {
    return new Promise((resolve, reject) => {
      const xhr = new XMLHttpRequest();
+      xhrMapRef.current.set(item.id, xhr);
+      
      const formData = new FormData();
      formData.append('file', item.file);
      if (tag) {
@@ -219,6 +508,7 @@ export function DragDropUpload({
      });

      xhr.addEventListener('load', () => {
+        xhrMapRef.current.delete(item.id);
        if (xhr.status >= 200 && xhr.status < 300) {
          try {
            const result = JSON.parse(xhr.responseText) as UploadResult;
@@ -237,18 +527,24 @@ export function DragDropUpload({
      });

      xhr.addEventListener('error', () => {
+        xhrMapRef.current.delete(item.id);
        reject(new Error('Network error - check your connection'));
      });

      xhr.addEventListener('timeout', () => {
+        xhrMapRef.current.delete(item.id);
        reject(new Error('Upload timed out'));
      });

+      xhr.addEventListener('abort', () => {
+        xhrMapRef.current.delete(item.id);
+        reject(new Error('Upload cancelled'));
+      });
+
      xhr.open('POST', `/api/v1/project/${projectName}/${packageName}/upload`);
-      xhr.timeout = 300000; // 5 minute timeout
+      xhr.timeout = 300000;
      xhr.send(formData);

-      // Store xhr for potential cancellation
      setUploadQueue(prev => prev.map(u => 
        u.id === item.id 
          ? { ...u, status: 'uploading' as UploadStatus, startTime: Date.now() }
@@ -257,8 +553,16 @@ export function DragDropUpload({
    });
  }, [projectName, packageName, tag]);

-  // Process upload queue
+  const uploadFile = useCallback((item: UploadItem): Promise<UploadResult> => {
+    if (item.file.size >= CHUNKED_UPLOAD_THRESHOLD) {
+      return uploadFileChunked(item);
+    }
+    return uploadFileSimple(item);
+  }, [uploadFileChunked, uploadFileSimple]);
+
  const processQueue = useCallback(async () => {
+    if (!isOnline) return;
+    
    const pendingItems = uploadQueue.filter(item => item.status === 'pending');
    
    for (const item of pendingItems) {
@@ -281,43 +585,49 @@ export function DragDropUpload({
            ? { ...u, status: 'complete' as UploadStatus, progress: 100, artifactId: result.artifact_id }
            : u
        ));
-      } catch (err) {
-        const errorMessage = err instanceof Error ? err.message : 'Upload failed';
-        const shouldRetry = item.retryCount < maxRetries && 
-          (errorMessage.includes('Network') || errorMessage.includes('timeout'));
-
-        if (shouldRetry) {
-          // Exponential backoff retry
-          const delay = Math.pow(2, item.retryCount) * 1000;
-          setTimeout(() => {
-            setUploadQueue(prev => prev.map(u => 
-              u.id === item.id 
-                ? { ...u, status: 'pending' as UploadStatus, retryCount: u.retryCount + 1, progress: 0 }
-                : u
-            ));
-          }, delay);
-        } else {
+      } catch (err: unknown) {
+        const dedupErr = err as { deduplicated?: boolean; artifact_id?: string };
+        if (dedupErr.deduplicated && dedupErr.artifact_id) {
          setUploadQueue(prev => prev.map(u => 
            u.id === item.id 
-              ? { ...u, status: 'failed' as UploadStatus, error: errorMessage }
+              ? { ...u, status: 'complete' as UploadStatus, progress: 100, artifactId: dedupErr.artifact_id }
              : u
          ));
-          onUploadError?.(errorMessage);
+        } else {
+          const errorMessage = err instanceof Error ? err.message : 'Upload failed';
+          const shouldRetry = item.retryCount < maxRetries && 
+            (errorMessage.includes('Network') || errorMessage.includes('timeout'));
+
+          if (shouldRetry) {
+            const delay = Math.pow(2, item.retryCount) * 1000;
+            setTimeout(() => {
+              setUploadQueue(prev => prev.map(u => 
+                u.id === item.id 
+                  ? { ...u, status: 'pending' as UploadStatus, retryCount: u.retryCount + 1, progress: 0 }
+                  : u
+              ));
+            }, delay);
+          } else {
+            setUploadQueue(prev => prev.map(u => 
+              u.id === item.id 
+                ? { ...u, status: 'failed' as UploadStatus, error: errorMessage }
+                : u
+            ));
+            onUploadError?.(errorMessage);
+          }
        }
      } finally {
        activeUploadsRef.current--;
      }
    }
-  }, [uploadQueue, maxConcurrentUploads, maxRetries, uploadFile, onUploadError]);
+  }, [uploadQueue, maxConcurrentUploads, maxRetries, uploadFile, onUploadError, isOnline]);

-  // Process queue when items are added or status changes
  useEffect(() => {
    const hasPending = uploadQueue.some(item => item.status === 'pending');
-    if (hasPending && activeUploadsRef.current < maxConcurrentUploads) {
+    if (hasPending && activeUploadsRef.current < maxConcurrentUploads && isOnline) {
      processQueue();
    }

-    // Notify when all uploads complete
    const allComplete = uploadQueue.length > 0 && 
      uploadQueue.every(item => item.status === 'complete' || item.status === 'failed');
    
@@ -333,7 +643,7 @@ export function DragDropUpload({
        onUploadComplete?.(completedResults);
      }
    }
-  }, [uploadQueue, maxConcurrentUploads, processQueue, onUploadComplete]);
+  }, [uploadQueue, maxConcurrentUploads, processQueue, onUploadComplete, isOnline]);

  // Drag event handlers
  const handleDragEnter = useCallback((e: React.DragEvent) => {
@@ -416,10 +726,17 @@ export function DragDropUpload({
  const completedCount = uploadQueue.filter(item => item.status === 'complete').length;
  const failedCount = uploadQueue.filter(item => item.status === 'failed').length;
  const uploadingCount = uploadQueue.filter(item => item.status === 'uploading').length;
+  const pausedCount = uploadQueue.filter(item => item.status === 'paused').length;

  return (
    <div className={`drag-drop-upload ${className}`}>
-      {/* Drop Zone */}
+      {!isOnline && (
+        <div className="offline-banner">
+          <WifiOffIcon />
+          <span>You're offline. Uploads will resume when connection is restored.</span>
+        </div>
+      )}
+
      <div
        className={`drop-zone ${isDragOver ? 'drop-zone--active' : ''}`}
        onDragEnter={handleDragEnter}
@@ -456,9 +773,11 @@ export function DragDropUpload({
        <div className="upload-queue">
          <div className="upload-queue__header">
            <span className="upload-queue__title">
-              {uploadingCount > 0 
-                ? `Uploading ${uploadingCount} of ${uploadQueue.length} files`
-                : `${completedCount} of ${uploadQueue.length} files uploaded`
+              {pausedCount > 0 && !isOnline
+                ? `${pausedCount} uploads paused (offline)`
+                : uploadingCount > 0 
+                  ? `Uploading ${uploadingCount} of ${uploadQueue.length} files`
+                  : `${completedCount} of ${uploadQueue.length} files uploaded`
              }
              {failedCount > 0 && ` (${failedCount} failed)`}
            </span>
@@ -493,6 +812,8 @@ export function DragDropUpload({
                <div className="upload-item__icon">
                  {item.status === 'complete' ? <CheckIcon /> : 
                   item.status === 'failed' ? <ErrorIcon /> : 
+                   item.status === 'paused' ? <PauseIcon /> :
+                   item.status === 'validating' ? <SpinnerIcon /> :
                   <FileIcon />}
                </div>
                
@@ -525,6 +846,9 @@ export function DragDropUpload({
                    {item.retryCount > 0 && item.status === 'uploading' && (
                      <span className="upload-item__retry-count">Retry {item.retryCount}</span>
                    )}
+                    {item.status === 'validating' && (
+                      <span className="upload-item__validating">Computing hash...</span>
+                    )}
                  </div>
                  
                  {item.status === 'uploading' && (
@@ -538,7 +862,7 @@ export function DragDropUpload({
                </div>

                <div className="upload-item__actions">
-                  {item.status === 'failed' && (
+                  {(item.status === 'failed' || (item.status === 'paused' && isOnline)) && (
                    <button 
                      className="upload-item__btn upload-item__btn--retry"
                      onClick={() => retryItem(item.id)}
@@ -548,7 +872,7 @@ export function DragDropUpload({
                      <RetryIcon />
                    </button>
                  )}
-                  {(item.status === 'complete' || item.status === 'failed' || item.status === 'pending') && (
+                  {(item.status === 'complete' || item.status === 'failed' || item.status === 'pending' || item.status === 'paused') && (
                    <button 
                      className="upload-item__btn upload-item__btn--remove"
                      onClick={() => removeItem(item.id)}