Merge branch 'fix/cleanup-and-pod-naming' into 'main'

Cleanup: improve pod naming, remove dead code, update docs See merge request esv/bsf/bsf-integration/orchard/orchard-mvp!29
2026-01-14 14:47:11 -06:00
parent 1bb0c4e911 32162c4ec7
commit e8f26e9976
7 changed files with 122 additions and 158 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,13 +26,9 @@ stages:
  - deploy

 kics:
-  allow_failure: true
  variables:
    KICS_CONFIG: kics.config

-hadolint:
-  allow_failure: true
-
 # Post-deployment integration tests template
 .integration_test_template: &integration_test_template
  stage: deploy  # Runs in deploy stage, but after deployment due to 'needs'
@@ -179,7 +175,7 @@ frontend_tests:
 # Shared deploy configuration
 .deploy_template: &deploy_template
  stage: deploy
-  needs: [build_image]
+  needs: [build_image, kics, hadolint, python_tests, frontend_tests]
  image: deps.global.bsf.tools/registry-1.docker.io/alpine/k8s:1.29.12

 .helm_setup: &helm_setup
@@ -250,7 +246,7 @@ deploy_stage:
        --set image.tag=git.linux-amd64-$CI_COMMIT_SHA \
        --wait \
        --timeout 5m
-    - kubectl rollout status deployment/orchard-stage -n $NAMESPACE --timeout=5m
+    - kubectl rollout status deployment/orchard-stage-server -n $NAMESPACE --timeout=5m
    - *verify_deployment
  environment:
    name: stage
@@ -285,7 +281,7 @@ deploy_feature:
        --set minioIngress.tls.secretName=minio-$CI_COMMIT_REF_SLUG-tls \
        --wait \
        --timeout 5m
-    - kubectl rollout status deployment/orchard-$CI_COMMIT_REF_SLUG -n $NAMESPACE --timeout=5m
+    - kubectl rollout status deployment/orchard-$CI_COMMIT_REF_SLUG-server -n $NAMESPACE --timeout=5m
    - export BASE_URL="https://orchard-$CI_COMMIT_REF_SLUG.common.global.bsf.tools"
    - *verify_deployment
  environment:
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added `deploy_feature` job with dynamic hostnames and unique release names (#51)
 - Added `cleanup_feature` job with `on_stop` for automatic cleanup on merge (#51)
 - Added `values-dev.yaml` Helm values for lightweight ephemeral environments (#51)
+- Added main branch deployment to stage environment (#51)
+- Added post-deployment integration tests (#51)
+- Added internal proxy configuration for npm, pip, helm, and apt (#51)
+
+### Changed
+- Improved pod naming: Orchard pods now named `orchard-{env}-server-*` for clarity (#51)
+
+### Fixed
+- Fixed `cleanup_feature` job failing when branch is deleted (`GIT_STRATEGY: none`) (#51)
+- Fixed gitleaks false positives with fingerprints for historical commits (#51)
+- Fixed integration tests running when deploy fails (`when: on_success`) (#51)
+
+### Removed
+- Removed unused `store_streaming()` method from storage.py (#51)

 ## [0.4.0] - 2026-01-12
 ### Added
--- a/README.md
+++ b/README.md
@@ -46,6 +46,12 @@ Orchard is a centralized binary artifact storage system that provides content-ad
  - `.whl` - Python wheels (name, version, author)
  - `.jar` - Java JARs (manifest info, Maven coordinates)
  - `.zip` - ZIP files (file count, uncompressed size)
+- **Authentication** - Multiple authentication methods:
+  - Session-based login with username/password
+  - API keys for programmatic access (`orch_` prefixed tokens)
+  - OIDC integration for SSO
+  - Admin user management
+- **Garbage Collection** - Clean up orphaned artifacts (ref_count=0) via admin API

 ### API Endpoints

@@ -522,15 +528,48 @@ Configuration is provided via environment variables prefixed with `ORCHARD_`:
 | `ORCHARD_DOWNLOAD_MODE` | Download mode: `presigned`, `redirect`, or `proxy` | `presigned` |
 | `ORCHARD_PRESIGNED_URL_EXPIRY` | Presigned URL expiry in seconds | `3600` |

+## CI/CD Pipeline
+
+The GitLab CI/CD pipeline automates building, testing, and deploying Orchard.
+
+### Pipeline Stages
+
+| Stage | Jobs | Description |
+|-------|------|-------------|
+| lint | `kics`, `hadolint`, `secrets` | Security and code quality scanning |
+| build | `build_image` | Build and push Docker image |
+| test | `python_tests`, `frontend_tests` | Run unit tests with coverage |
+| deploy | `deploy_stage`, `deploy_feature` | Deploy to Kubernetes |
+| deploy | `integration_test_*` | Post-deployment integration tests |
+
+### Environments
+
+| Environment | Branch | Namespace | URL |
+|-------------|--------|-----------|-----|
+| Stage | `main` | `orch-stage-namespace` | `orchard-stage.common.global.bsf.tools` |
+| Feature | `*` (non-main) | `orch-dev-namespace` | `orchard-{branch}.common.global.bsf.tools` |
+
+### Feature Branch Workflow
+
+1. Push a feature branch
+2. Pipeline builds, tests, and deploys to isolated environment
+3. Integration tests run against the deployed environment
+4. GitLab UI shows environment link for manual testing
+5. On merge to main, environment is automatically cleaned up
+6. Environments also auto-expire after 1 week if branch is not deleted
+
+### Manual Cleanup
+
+Feature environments can be manually cleaned up via:
+- GitLab UI: Environments → Stop environment
+- CLI: `helm uninstall orchard-{branch} -n orch-dev-namespace`
+
 ## Kubernetes Deployment

 ### Using Helm

 ```bash
-# Add Bitnami repo for dependencies
-helm repo add bitnami https://charts.bitnami.com/bitnami
-
-# Update dependencies
+# Update dependencies (uses internal OCI registry)
 cd helm/orchard
 helm dependency update

@@ -593,10 +632,16 @@ The following features are planned but not yet implemented:
 - [ ] Export/Import for air-gapped systems
 - [ ] Consumer notification
 - [ ] Automated update propagation
- [ ] OIDC/SAML authentication
- [ ] API key management
+- [ ] SAML authentication
 - [ ] Redis caching layer
- [ ] Garbage collection for orphaned artifacts
+- [ ] Download integrity verification (see `docs/design/integrity-verification.md`)
+
+### Recently Implemented
+
+- [x] OIDC authentication
+- [x] API key management
+- [x] Garbage collection for orphaned artifacts
+- [x] User authentication with sessions

 ## License

--- a/backend/app/storage.py
+++ b/backend/app/storage.py
@@ -6,7 +6,6 @@ from typing import (
    Optional,
    Dict,
    Any,
-    Generator,
    NamedTuple,
    Protocol,
    runtime_checkable,
@@ -511,127 +510,6 @@ class S3Storage:
            )
            raise

-    def store_streaming(self, chunks: Generator[bytes, None, None]) -> StorageResult:
-        """
-        Store a file from a stream of chunks.
-        First accumulates to compute hash, then uploads.
-        For truly large files, consider using initiate_resumable_upload instead.
-        """
-        # Accumulate chunks and compute all hashes
-        sha256_hasher = hashlib.sha256()
-        md5_hasher = hashlib.md5()
-        sha1_hasher = hashlib.sha1()
-        all_chunks = []
-        size = 0
-
-        for chunk in chunks:
-            sha256_hasher.update(chunk)
-            md5_hasher.update(chunk)
-            sha1_hasher.update(chunk)
-            all_chunks.append(chunk)
-            size += len(chunk)
-
-        sha256_hash = sha256_hasher.hexdigest()
-        md5_hash = md5_hasher.hexdigest()
-        sha1_hash = sha1_hasher.hexdigest()
-        s3_key = f"fruits/{sha256_hash[:2]}/{sha256_hash[2:4]}/{sha256_hash}"
-        s3_etag = None
-
-        # Check if already exists
-        if self._exists(s3_key):
-            obj_info = self.get_object_info(s3_key)
-            s3_etag = obj_info.get("etag", "").strip('"') if obj_info else None
-            return StorageResult(
-                sha256=sha256_hash,
-                size=size,
-                s3_key=s3_key,
-                md5=md5_hash,
-                sha1=sha1_hash,
-                s3_etag=s3_etag,
-                already_existed=True,
-            )
-
-        # Upload based on size
-        if size < MULTIPART_THRESHOLD:
-            content = b"".join(all_chunks)
-            response = self.client.put_object(
-                Bucket=self.bucket, Key=s3_key, Body=content
-            )
-            s3_etag = response.get("ETag", "").strip('"')
-        else:
-            # Use multipart for large files
-            mpu = self.client.create_multipart_upload(Bucket=self.bucket, Key=s3_key)
-            upload_id = mpu["UploadId"]
-
-            try:
-                parts = []
-                part_number = 1
-                buffer = b""
-
-                for chunk in all_chunks:
-                    buffer += chunk
-                    while len(buffer) >= MULTIPART_CHUNK_SIZE:
-                        part_data = buffer[:MULTIPART_CHUNK_SIZE]
-                        buffer = buffer[MULTIPART_CHUNK_SIZE:]
-
-                        response = self.client.upload_part(
-                            Bucket=self.bucket,
-                            Key=s3_key,
-                            UploadId=upload_id,
-                            PartNumber=part_number,
-                            Body=part_data,
-                        )
-                        parts.append(
-                            {
-                                "PartNumber": part_number,
-                                "ETag": response["ETag"],
-                            }
-                        )
-                        part_number += 1
-
-                # Upload remaining buffer
-                if buffer:
-                    response = self.client.upload_part(
-                        Bucket=self.bucket,
-                        Key=s3_key,
-                        UploadId=upload_id,
-                        PartNumber=part_number,
-                        Body=buffer,
-                    )
-                    parts.append(
-                        {
-                            "PartNumber": part_number,
-                            "ETag": response["ETag"],
-                        }
-                    )
-
-                complete_response = self.client.complete_multipart_upload(
-                    Bucket=self.bucket,
-                    Key=s3_key,
-                    UploadId=upload_id,
-                    MultipartUpload={"Parts": parts},
-                )
-                s3_etag = complete_response.get("ETag", "").strip('"')
-
-            except Exception as e:
-                logger.error(f"Streaming multipart upload failed: {e}")
-                self.client.abort_multipart_upload(
-                    Bucket=self.bucket,
-                    Key=s3_key,
-                    UploadId=upload_id,
-                )
-                raise
-
-        return StorageResult(
-            sha256=sha256_hash,
-            size=size,
-            s3_key=s3_key,
-            md5=md5_hash,
-            sha1=sha1_hash,
-            s3_etag=s3_etag,
-            already_existed=False,
-        )
-
    def initiate_resumable_upload(self, expected_hash: str) -> Dict[str, Any]:
        """
        Initiate a resumable upload session.
--- a/docker-compose.local.yml
+++ b/docker-compose.local.yml
@@ -46,8 +46,11 @@ services:
      - no-new-privileges:true
    cap_drop:
      - ALL
-    mem_limit: 1g
-    cpus: 1.0
+    deploy:
+      resources:
+        limits:
+          cpus: '1.0'
+          memory: 1G

  postgres:
    image: postgres:16-alpine
@@ -72,8 +75,11 @@ services:
      - no-new-privileges:true
    cap_drop:
      - ALL
-    mem_limit: 512m
-    cpus: 0.5
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 512M

  minio:
    image: minio/minio:latest
@@ -98,8 +104,11 @@ services:
      - no-new-privileges:true
    cap_drop:
      - ALL
-    mem_limit: 512m
-    cpus: 0.5
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 512M

  minio-init:
    image: minio/mc:latest
@@ -119,8 +128,11 @@ services:
      - no-new-privileges:true
    cap_drop:
      - ALL
-    mem_limit: 128m
-    cpus: 0.25
+    deploy:
+      resources:
+        limits:
+          cpus: '0.25'
+          memory: 128M

  redis:
    image: redis:7-alpine
@@ -141,8 +153,11 @@ services:
      - no-new-privileges:true
    cap_drop:
      - ALL
-    mem_limit: 256m
-    cpus: 0.25
+    deploy:
+      resources:
+        limits:
+          cpus: '0.25'
+          memory: 256M

 volumes:
  postgres-data-local:
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -44,8 +44,11 @@ services:
      - no-new-privileges:true
    cap_drop:
      - ALL
-    mem_limit: 1g
-    cpus: 1.0
+    deploy:
+      resources:
+        limits:
+          cpus: '1.0'
+          memory: 1G

  postgres:
    image: containers.global.bsf.tools/postgres:16-alpine
@@ -70,8 +73,11 @@ services:
      - no-new-privileges:true
    cap_drop:
      - ALL
-    mem_limit: 512m
-    cpus: 0.5
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 512M

  minio:
    image: containers.global.bsf.tools/minio/minio:latest
@@ -96,8 +102,11 @@ services:
      - no-new-privileges:true
    cap_drop:
      - ALL
-    mem_limit: 512m
-    cpus: 0.5
+    deploy:
+      resources:
+        limits:
+          cpus: '0.5'
+          memory: 512M

  minio-init:
    image: containers.global.bsf.tools/minio/mc:latest
@@ -117,8 +126,11 @@ services:
      - no-new-privileges:true
    cap_drop:
      - ALL
-    mem_limit: 128m
-    cpus: 0.25
+    deploy:
+      resources:
+        limits:
+          cpus: '0.25'
+          memory: 128M

  redis:
    image: containers.global.bsf.tools/redis:7-alpine
@@ -139,8 +151,11 @@ services:
      - no-new-privileges:true
    cap_drop:
      - ALL
-    mem_limit: 256m
-    cpus: 0.25
+    deploy:
+      resources:
+        limits:
+          cpus: '0.25'
+          memory: 256M

 volumes:
  postgres-data:
--- a/helm/orchard/templates/_helpers.tpl
+++ b/helm/orchard/templates/_helpers.tpl
@@ -7,6 +7,7 @@ Expand the name of the chart.

 {{/*
 Create a default fully qualified app name.
+Appends "-server" to distinguish from subcharts (minio, postgresql, redis).
 */}}
 {{- define "orchard.fullname" -}}
 {{- if .Values.fullnameOverride }}
@@ -14,9 +15,9 @@ Create a default fully qualified app name.
 {{- else }}
 {{- $name := default .Chart.Name .Values.nameOverride }}
 {{- if contains $name .Release.Name }}
-{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- printf "%s-server" .Release.Name | trunc 63 | trimSuffix "-" }}
 {{- else }}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- printf "%s-%s-server" .Release.Name $name | trunc 63 | trimSuffix "-" }}
 {{- end }}
 {{- end }}
 {{- end }}