From 32162c4ec79430c508b9c910b3bb09e608d1d05b Mon Sep 17 00:00:00 2001 From: Mondo Diaz Date: Wed, 14 Jan 2026 14:47:11 -0600 Subject: [PATCH] Cleanup: improve pod naming, remove dead code, update docs --- .gitlab-ci.yml | 10 +-- CHANGELOG.md | 14 ++++ README.md | 59 ++++++++++++-- backend/app/storage.py | 122 ---------------------------- docker-compose.local.yml | 35 +++++--- docker-compose.yml | 35 +++++--- helm/orchard/templates/_helpers.tpl | 5 +- 7 files changed, 122 insertions(+), 158 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2563a7e..e06016a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,13 +26,9 @@ stages: - deploy kics: - allow_failure: true variables: KICS_CONFIG: kics.config -hadolint: - allow_failure: true - # Post-deployment integration tests template .integration_test_template: &integration_test_template stage: deploy # Runs in deploy stage, but after deployment due to 'needs' @@ -179,7 +175,7 @@ frontend_tests: # Shared deploy configuration .deploy_template: &deploy_template stage: deploy - needs: [build_image] + needs: [build_image, kics, hadolint, python_tests, frontend_tests] image: deps.global.bsf.tools/registry-1.docker.io/alpine/k8s:1.29.12 .helm_setup: &helm_setup @@ -250,7 +246,7 @@ deploy_stage: --set image.tag=git.linux-amd64-$CI_COMMIT_SHA \ --wait \ --timeout 5m - - kubectl rollout status deployment/orchard-stage -n $NAMESPACE --timeout=5m + - kubectl rollout status deployment/orchard-stage-server -n $NAMESPACE --timeout=5m - *verify_deployment environment: name: stage @@ -285,7 +281,7 @@ deploy_feature: --set minioIngress.tls.secretName=minio-$CI_COMMIT_REF_SLUG-tls \ --wait \ --timeout 5m - - kubectl rollout status deployment/orchard-$CI_COMMIT_REF_SLUG -n $NAMESPACE --timeout=5m + - kubectl rollout status deployment/orchard-$CI_COMMIT_REF_SLUG-server -n $NAMESPACE --timeout=5m - export BASE_URL="https://orchard-$CI_COMMIT_REF_SLUG.common.global.bsf.tools" - *verify_deployment environment: diff --git a/CHANGELOG.md b/CHANGELOG.md index d95fc17..1e0979b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `deploy_feature` job with dynamic hostnames and unique release names (#51) - Added `cleanup_feature` job with `on_stop` for automatic cleanup on merge (#51) - Added `values-dev.yaml` Helm values for lightweight ephemeral environments (#51) +- Added main branch deployment to stage environment (#51) +- Added post-deployment integration tests (#51) +- Added internal proxy configuration for npm, pip, helm, and apt (#51) + +### Changed +- Improved pod naming: Orchard pods now named `orchard-{env}-server-*` for clarity (#51) + +### Fixed +- Fixed `cleanup_feature` job failing when branch is deleted (`GIT_STRATEGY: none`) (#51) +- Fixed gitleaks false positives with fingerprints for historical commits (#51) +- Fixed integration tests running when deploy fails (`when: on_success`) (#51) + +### Removed +- Removed unused `store_streaming()` method from storage.py (#51) ## [0.4.0] - 2026-01-12 ### Added diff --git a/README.md b/README.md index 796d20d..4d711c0 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,12 @@ Orchard is a centralized binary artifact storage system that provides content-ad - `.whl` - Python wheels (name, version, author) - `.jar` - Java JARs (manifest info, Maven coordinates) - `.zip` - ZIP files (file count, uncompressed size) +- **Authentication** - Multiple authentication methods: + - Session-based login with username/password + - API keys for programmatic access (`orch_` prefixed tokens) + - OIDC integration for SSO + - Admin user management +- **Garbage Collection** - Clean up orphaned artifacts (ref_count=0) via admin API ### API Endpoints @@ -522,15 +528,48 @@ Configuration is provided via environment variables prefixed with `ORCHARD_`: | `ORCHARD_DOWNLOAD_MODE` | Download mode: `presigned`, `redirect`, or `proxy` | `presigned` | | `ORCHARD_PRESIGNED_URL_EXPIRY` | Presigned URL expiry in seconds | `3600` | +## CI/CD Pipeline + +The GitLab CI/CD pipeline automates building, testing, and deploying Orchard. + +### Pipeline Stages + +| Stage | Jobs | Description | +|-------|------|-------------| +| lint | `kics`, `hadolint`, `secrets` | Security and code quality scanning | +| build | `build_image` | Build and push Docker image | +| test | `python_tests`, `frontend_tests` | Run unit tests with coverage | +| deploy | `deploy_stage`, `deploy_feature` | Deploy to Kubernetes | +| deploy | `integration_test_*` | Post-deployment integration tests | + +### Environments + +| Environment | Branch | Namespace | URL | +|-------------|--------|-----------|-----| +| Stage | `main` | `orch-stage-namespace` | `orchard-stage.common.global.bsf.tools` | +| Feature | `*` (non-main) | `orch-dev-namespace` | `orchard-{branch}.common.global.bsf.tools` | + +### Feature Branch Workflow + +1. Push a feature branch +2. Pipeline builds, tests, and deploys to isolated environment +3. Integration tests run against the deployed environment +4. GitLab UI shows environment link for manual testing +5. On merge to main, environment is automatically cleaned up +6. Environments also auto-expire after 1 week if branch is not deleted + +### Manual Cleanup + +Feature environments can be manually cleaned up via: +- GitLab UI: Environments → Stop environment +- CLI: `helm uninstall orchard-{branch} -n orch-dev-namespace` + ## Kubernetes Deployment ### Using Helm ```bash -# Add Bitnami repo for dependencies -helm repo add bitnami https://charts.bitnami.com/bitnami - -# Update dependencies +# Update dependencies (uses internal OCI registry) cd helm/orchard helm dependency update @@ -593,10 +632,16 @@ The following features are planned but not yet implemented: - [ ] Export/Import for air-gapped systems - [ ] Consumer notification - [ ] Automated update propagation -- [ ] OIDC/SAML authentication -- [ ] API key management +- [ ] SAML authentication - [ ] Redis caching layer -- [ ] Garbage collection for orphaned artifacts +- [ ] Download integrity verification (see `docs/design/integrity-verification.md`) + +### Recently Implemented + +- [x] OIDC authentication +- [x] API key management +- [x] Garbage collection for orphaned artifacts +- [x] User authentication with sessions ## License diff --git a/backend/app/storage.py b/backend/app/storage.py index 672d841..ca3ffe3 100644 --- a/backend/app/storage.py +++ b/backend/app/storage.py @@ -6,7 +6,6 @@ from typing import ( Optional, Dict, Any, - Generator, NamedTuple, Protocol, runtime_checkable, @@ -511,127 +510,6 @@ class S3Storage: ) raise - def store_streaming(self, chunks: Generator[bytes, None, None]) -> StorageResult: - """ - Store a file from a stream of chunks. - First accumulates to compute hash, then uploads. - For truly large files, consider using initiate_resumable_upload instead. - """ - # Accumulate chunks and compute all hashes - sha256_hasher = hashlib.sha256() - md5_hasher = hashlib.md5() - sha1_hasher = hashlib.sha1() - all_chunks = [] - size = 0 - - for chunk in chunks: - sha256_hasher.update(chunk) - md5_hasher.update(chunk) - sha1_hasher.update(chunk) - all_chunks.append(chunk) - size += len(chunk) - - sha256_hash = sha256_hasher.hexdigest() - md5_hash = md5_hasher.hexdigest() - sha1_hash = sha1_hasher.hexdigest() - s3_key = f"fruits/{sha256_hash[:2]}/{sha256_hash[2:4]}/{sha256_hash}" - s3_etag = None - - # Check if already exists - if self._exists(s3_key): - obj_info = self.get_object_info(s3_key) - s3_etag = obj_info.get("etag", "").strip('"') if obj_info else None - return StorageResult( - sha256=sha256_hash, - size=size, - s3_key=s3_key, - md5=md5_hash, - sha1=sha1_hash, - s3_etag=s3_etag, - already_existed=True, - ) - - # Upload based on size - if size < MULTIPART_THRESHOLD: - content = b"".join(all_chunks) - response = self.client.put_object( - Bucket=self.bucket, Key=s3_key, Body=content - ) - s3_etag = response.get("ETag", "").strip('"') - else: - # Use multipart for large files - mpu = self.client.create_multipart_upload(Bucket=self.bucket, Key=s3_key) - upload_id = mpu["UploadId"] - - try: - parts = [] - part_number = 1 - buffer = b"" - - for chunk in all_chunks: - buffer += chunk - while len(buffer) >= MULTIPART_CHUNK_SIZE: - part_data = buffer[:MULTIPART_CHUNK_SIZE] - buffer = buffer[MULTIPART_CHUNK_SIZE:] - - response = self.client.upload_part( - Bucket=self.bucket, - Key=s3_key, - UploadId=upload_id, - PartNumber=part_number, - Body=part_data, - ) - parts.append( - { - "PartNumber": part_number, - "ETag": response["ETag"], - } - ) - part_number += 1 - - # Upload remaining buffer - if buffer: - response = self.client.upload_part( - Bucket=self.bucket, - Key=s3_key, - UploadId=upload_id, - PartNumber=part_number, - Body=buffer, - ) - parts.append( - { - "PartNumber": part_number, - "ETag": response["ETag"], - } - ) - - complete_response = self.client.complete_multipart_upload( - Bucket=self.bucket, - Key=s3_key, - UploadId=upload_id, - MultipartUpload={"Parts": parts}, - ) - s3_etag = complete_response.get("ETag", "").strip('"') - - except Exception as e: - logger.error(f"Streaming multipart upload failed: {e}") - self.client.abort_multipart_upload( - Bucket=self.bucket, - Key=s3_key, - UploadId=upload_id, - ) - raise - - return StorageResult( - sha256=sha256_hash, - size=size, - s3_key=s3_key, - md5=md5_hash, - sha1=sha1_hash, - s3_etag=s3_etag, - already_existed=False, - ) - def initiate_resumable_upload(self, expected_hash: str) -> Dict[str, Any]: """ Initiate a resumable upload session. diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 3bfc6db..3792e3e 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -46,8 +46,11 @@ services: - no-new-privileges:true cap_drop: - ALL - mem_limit: 1g - cpus: 1.0 + deploy: + resources: + limits: + cpus: '1.0' + memory: 1G postgres: image: postgres:16-alpine @@ -72,8 +75,11 @@ services: - no-new-privileges:true cap_drop: - ALL - mem_limit: 512m - cpus: 0.5 + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M minio: image: minio/minio:latest @@ -98,8 +104,11 @@ services: - no-new-privileges:true cap_drop: - ALL - mem_limit: 512m - cpus: 0.5 + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M minio-init: image: minio/mc:latest @@ -119,8 +128,11 @@ services: - no-new-privileges:true cap_drop: - ALL - mem_limit: 128m - cpus: 0.25 + deploy: + resources: + limits: + cpus: '0.25' + memory: 128M redis: image: redis:7-alpine @@ -141,8 +153,11 @@ services: - no-new-privileges:true cap_drop: - ALL - mem_limit: 256m - cpus: 0.25 + deploy: + resources: + limits: + cpus: '0.25' + memory: 256M volumes: postgres-data-local: diff --git a/docker-compose.yml b/docker-compose.yml index d0ba98f..00dcc73 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,8 +44,11 @@ services: - no-new-privileges:true cap_drop: - ALL - mem_limit: 1g - cpus: 1.0 + deploy: + resources: + limits: + cpus: '1.0' + memory: 1G postgres: image: containers.global.bsf.tools/postgres:16-alpine @@ -70,8 +73,11 @@ services: - no-new-privileges:true cap_drop: - ALL - mem_limit: 512m - cpus: 0.5 + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M minio: image: containers.global.bsf.tools/minio/minio:latest @@ -96,8 +102,11 @@ services: - no-new-privileges:true cap_drop: - ALL - mem_limit: 512m - cpus: 0.5 + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M minio-init: image: containers.global.bsf.tools/minio/mc:latest @@ -117,8 +126,11 @@ services: - no-new-privileges:true cap_drop: - ALL - mem_limit: 128m - cpus: 0.25 + deploy: + resources: + limits: + cpus: '0.25' + memory: 128M redis: image: containers.global.bsf.tools/redis:7-alpine @@ -139,8 +151,11 @@ services: - no-new-privileges:true cap_drop: - ALL - mem_limit: 256m - cpus: 0.25 + deploy: + resources: + limits: + cpus: '0.25' + memory: 256M volumes: postgres-data: diff --git a/helm/orchard/templates/_helpers.tpl b/helm/orchard/templates/_helpers.tpl index 541c9df..211d33e 100644 --- a/helm/orchard/templates/_helpers.tpl +++ b/helm/orchard/templates/_helpers.tpl @@ -7,6 +7,7 @@ Expand the name of the chart. {{/* Create a default fully qualified app name. +Appends "-server" to distinguish from subcharts (minio, postgresql, redis). */}} {{- define "orchard.fullname" -}} {{- if .Values.fullnameOverride }} @@ -14,9 +15,9 @@ Create a default fully qualified app name. {{- else }} {{- $name := default .Chart.Name .Values.nameOverride }} {{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- printf "%s-server" .Release.Name | trunc 63 | trimSuffix "-" }} {{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- printf "%s-%s-server" .Release.Name $name | trunc 63 | trimSuffix "-" }} {{- end }} {{- end }} {{- end }}