Add comprehensive upload/download tests and streaming enhancements (#38, #40, #42, #43)

.gitlab-ci.yml

@@ -8,6 +8,11 @@ variables:
   PROSPER_VERSION: v0.64.1
   # Use internal PyPI proxy instead of public internet
   PIP_INDEX_URL: https://deps.global.bsf.tools/artifactory/api/pypi/pypi.org/simple
+  # Environment URLs (used by deploy and test jobs)
+  STAGE_URL: https://orchard-stage.common.global.bsf.tools
+  PROD_URL: https://orchard.common.global.bsf.tools
+  # Shared pip cache directory
+  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.pip-cache"
 
 # Prevent duplicate pipelines for MRs
 workflow:
@@ -29,11 +34,47 @@ kics:
   variables:
     KICS_CONFIG: kics.config
 
-# Post-deployment integration tests template
+# Full integration test suite template (for feature/stage deployments)
+# Runs the complete pytest integration test suite against the deployed environment
 .integration_test_template: &integration_test_template
   stage: deploy  # Runs in deploy stage, but after deployment due to 'needs'
   image: deps.global.bsf.tools/docker/python:3.12-slim
-  timeout: 10m
+  timeout: 20m  # Full suite takes longer than smoke tests
+  interruptible: true  # Cancel if new pipeline starts
+  retry: 1  # Retry once on failure (network flakiness)
+  cache:
+    key: pip-$CI_COMMIT_REF_SLUG
+    paths:
+      - .pip-cache/
+    policy: pull-push
+  before_script:
+    - pip install --index-url "$PIP_INDEX_URL" -r backend/requirements.txt
+    - pip install --index-url "$PIP_INDEX_URL" pytest pytest-asyncio httpx
+  script:
+    - cd backend
+    # Run full integration test suite, excluding:
+    # - large/slow tests
+    # - requires_direct_s3 tests (can't access MinIO from outside K8s cluster)
+    # ORCHARD_TEST_URL tells the tests which server to connect to
+    # Note: Auth tests work because dev/stage deployments have relaxed rate limits
+    - |
+      python -m pytest tests/integration/ -v \
+        --junitxml=integration-report.xml \
+        -m "not large and not slow and not requires_direct_s3" \
+        --tb=short
+  artifacts:
+    when: always
+    expire_in: 1 week
+    paths:
+      - backend/integration-report.xml
+    reports:
+      junit: backend/integration-report.xml
+
+# Lightweight smoke test template (for production - no test data creation)
+.smoke_test_template: &smoke_test_template
+  stage: deploy
+  image: deps.global.bsf.tools/docker/python:3.12-slim
+  timeout: 5m
   before_script:
     - pip install --index-url "$PIP_INDEX_URL" httpx
   script:
@@ -43,12 +84,12 @@ kics:
       import os
       import sys
 
-      BASE_URL = os.environ.get("BASE_URL")
+      BASE_URL = os.environ.get("ORCHARD_TEST_URL")
       if not BASE_URL:
-          print("ERROR: BASE_URL not set")
+          print("ERROR: ORCHARD_TEST_URL not set")
           sys.exit(1)
 
-      print(f"Running integration tests against {BASE_URL}")
+      print(f"Running smoke tests against {BASE_URL}")
       client = httpx.Client(base_url=BASE_URL, timeout=30.0)
 
       errors = []
@@ -86,38 +127,37 @@ kics:
               print(f"  FAIL: {e}")
           sys.exit(1)
       else:
-          print("SUCCESS: All integration tests passed!")
+          print("SUCCESS: All smoke tests passed!")
           sys.exit(0)
       PYTEST_SCRIPT
 
-# Integration tests for stage deployment
+# Integration tests for stage deployment (full suite)
 integration_test_stage:
   <<: *integration_test_template
   needs: [deploy_stage]
   variables:
-    BASE_URL: https://orchard-stage.common.global.bsf.tools
+    ORCHARD_TEST_URL: $STAGE_URL
   rules:
     - if: '$CI_COMMIT_BRANCH == "main"'
       when: on_success
 
-# Integration tests for feature deployment
+# Integration tests for feature deployment (full suite)
 integration_test_feature:
   <<: *integration_test_template
   needs: [deploy_feature]
   variables:
-    BASE_URL: https://orchard-$CI_COMMIT_REF_SLUG.common.global.bsf.tools
+    ORCHARD_TEST_URL: https://orchard-$CI_COMMIT_REF_SLUG.common.global.bsf.tools
   rules:
     - if: '$CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != "main"'
       when: on_success
 
-# Run Python backend tests
-python_tests:
+# Run Python backend unit tests
+python_unit_tests:
   stage: test
   needs: []  # Run in parallel with build
   image: deps.global.bsf.tools/docker/python:3.12-slim
   timeout: 15m
-  variables:
-    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.pip-cache"
+  interruptible: true  # Cancel if new pipeline starts
   cache:
     key: pip-$CI_COMMIT_REF_SLUG
     paths:
@@ -128,7 +168,7 @@ python_tests:
     - pip install --index-url "$PIP_INDEX_URL" pytest pytest-asyncio pytest-cov httpx
   script:
     - cd backend
-    # Only run unit tests - integration tests require Docker Compose services
+    # Run unit tests (integration tests run post-deployment against live environment)
     - python -m pytest tests/unit/ -v --cov=app --cov-report=term --cov-report=xml:coverage.xml --cov-report=html:coverage_html --junitxml=pytest-report.xml
   artifacts:
     when: always
@@ -150,6 +190,7 @@ frontend_tests:
   needs: []  # Run in parallel with build
   image: deps.global.bsf.tools/docker/node:20-alpine
   timeout: 15m
+  interruptible: true  # Cancel if new pipeline starts
   cache:
     key: npm-$CI_COMMIT_REF_SLUG
     paths:
@@ -175,7 +216,7 @@ frontend_tests:
 # Shared deploy configuration
 .deploy_template: &deploy_template
   stage: deploy
-  needs: [build_image, test_image, kics, hadolint, python_tests, frontend_tests, secrets, app_deps_scan, cve_scan, cve_sbom_analysis, app_sbom_analysis]
+  needs: [build_image, test_image, kics, hadolint, python_unit_tests, frontend_tests, secrets, app_deps_scan, cve_scan, cve_sbom_analysis, app_sbom_analysis]
   image: deps.global.bsf.tools/registry-1.docker.io/alpine/k8s:1.29.12
 
 .helm_setup: &helm_setup
@@ -184,47 +225,21 @@ frontend_tests:
   # OCI-based charts from internal registry - no repo add needed
   - helm dependency update
 
+# Simplified deployment verification - just health check
+# Full API/frontend checks are done by integration tests post-deployment
 .verify_deployment: &verify_deployment |
   echo "=== Waiting for health endpoint (certs may take a few minutes) ==="
   for i in $(seq 1 30); do
     if curl -sf --max-time 10 "$BASE_URL/health" > /dev/null 2>&1; then
       echo "Health check passed!"
-      break
+      echo "Deployment URL: $BASE_URL"
+      exit 0
     fi
     echo "Attempt $i/30 - waiting 10s..."
     sleep 10
   done
-
-  # Verify health endpoint
-  echo ""
-  echo "=== Health Check ==="
-  curl -sf "$BASE_URL/health" || { echo "Health check failed"; exit 1; }
-  echo ""
-
-  # Verify API is responding
-  echo ""
-  echo "=== API Check (GET /api/v1/projects) ==="
-  HTTP_CODE=$(curl -sf -o /dev/null -w "%{http_code}" "$BASE_URL/api/v1/projects")
-  if [ "$HTTP_CODE" = "200" ]; then
-    echo "API responding: HTTP $HTTP_CODE"
-  else
-    echo "API check failed: HTTP $HTTP_CODE"
-    exit 1
-  fi
-
-  # Verify frontend is served
-  echo ""
-  echo "=== Frontend Check ==="
-  if curl -sf "$BASE_URL/" | grep -q "</html>"; then
-    echo "Frontend is being served"
-  else
-    echo "Frontend check failed"
-    exit 1
-  fi
-
-  echo ""
-  echo "=== All checks passed! ==="
-  echo "Deployment URL: $BASE_URL"
+  echo "Health check failed after 30 attempts"
+  exit 1
 
 # Deploy to stage (main branch)
 deploy_stage:
@@ -232,7 +247,7 @@ deploy_stage:
   variables:
     NAMESPACE: orch-stage-namespace
     VALUES_FILE: helm/orchard/values-stage.yaml
-    BASE_URL: https://orchard-stage.common.global.bsf.tools
+    BASE_URL: $STAGE_URL
   before_script:
     - kubectl config use-context esv/bsf/bsf-integration/orchard/orchard-mvp:orchard-stage
     - *helm_setup
@@ -251,7 +266,7 @@ deploy_stage:
     - *verify_deployment
   environment:
     name: stage
-    url: https://orchard-stage.common.global.bsf.tools
+    url: $STAGE_URL
     kubernetes:
       agent: esv/bsf/bsf-integration/orchard/orchard-mvp:orchard-stage
   rules:
@@ -297,10 +312,12 @@ deploy_feature:
     - if: '$CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != "main"'
       when: on_success
 
-# Cleanup feature branch deployment
+# Cleanup feature branch deployment (standalone - doesn't need deploy dependencies)
 cleanup_feature:
-  <<: *deploy_template
+  stage: deploy
   needs: []
+  image: deps.global.bsf.tools/registry-1.docker.io/alpine/k8s:1.29.12
+  timeout: 5m
   variables:
     NAMESPACE: orch-dev-namespace
     GIT_STRATEGY: none  # No source needed, branch may be deleted
@@ -329,7 +346,7 @@ deploy_prod:
   variables:
     NAMESPACE: orch-prod-namespace
     VALUES_FILE: helm/orchard/values-prod.yaml
-    BASE_URL: https://orchard.common.global.bsf.tools
+    BASE_URL: $PROD_URL
   before_script:
     - kubectl config use-context esv/bsf/bsf-integration/orchard/orchard-mvp:orchard-prod
     - *helm_setup
@@ -348,7 +365,7 @@ deploy_prod:
     - *verify_deployment
   environment:
     name: production
-    url: https://orchard.common.global.bsf.tools
+    url: $PROD_URL
     kubernetes:
       agent: esv/bsf/bsf-integration/orchard/orchard-mvp:orchard-prod
   rules:
@@ -357,12 +374,12 @@ deploy_prod:
       when: manual  # Require manual approval for prod
   allow_failure: false
 
-# Integration tests for production deployment
-integration_test_prod:
-  <<: *integration_test_template
+# Smoke tests for production deployment (read-only, no test data creation)
+smoke_test_prod:
+  <<: *smoke_test_template
   needs: [deploy_prod]
   variables:
-    BASE_URL: https://orchard.common.global.bsf.tools
+    ORCHARD_TEST_URL: $PROD_URL
   rules:
     - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/'
       when: on_success