Compare commits
19 Commits
199821b34d
...
fix/ci-pro
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0e01e17dfe | ||
|
|
ca46ab1ea0 | ||
|
|
a01c45cb64 | ||
|
|
133d9cbfd6 | ||
|
|
276b4f2743 | ||
|
|
67ac6bb3f8 | ||
|
|
b0bb3ed569 | ||
|
|
1ac75e1017 | ||
|
|
693613f111 | ||
|
|
9da4ae8c0d | ||
|
|
7ffdc64364 | ||
|
|
6abc0c88b0 | ||
|
|
e96dc5cde8 | ||
|
|
cba5bac383 | ||
|
|
535280a783 | ||
|
|
c9026e1950 | ||
|
|
fedbd95cf4 | ||
|
|
255e25d66d | ||
|
|
427d2fec70 |
164
.gitlab-ci.yml
164
.gitlab-ci.yml
@@ -11,6 +11,12 @@ variables:
|
||||
# Environment URLs (used by deploy and test jobs)
|
||||
STAGE_URL: https://orchard-stage.common.global.bsf.tools
|
||||
PROD_URL: https://orchard.common.global.bsf.tools
|
||||
# Stage environment AWS resources (used by reset job)
|
||||
STAGE_RDS_HOST: orchard-stage.cluster-cvw3jzjkozoc.us-gov-west-1.rds.amazonaws.com
|
||||
STAGE_RDS_DBNAME: postgres
|
||||
STAGE_SECRET_ARN: "arn:aws-us-gov:secretsmanager:us-gov-west-1:052673043337:secret:rds!cluster-a573672b-1a38-4665-a654-1b7df37b5297-IaeFQL"
|
||||
STAGE_S3_BUCKET: orchard-artifacts-stage
|
||||
AWS_REGION: us-gov-west-1
|
||||
# Shared pip cache directory
|
||||
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.pip-cache"
|
||||
|
||||
@@ -30,9 +36,68 @@ stages:
|
||||
- analyze
|
||||
- deploy
|
||||
|
||||
# Override Prosper template jobs to exclude tag pipelines
|
||||
# Tags only run deploy_prod and smoke_test_prod (image already built on main)
|
||||
build_image:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
test_image:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
hadolint:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
kics:
|
||||
variables:
|
||||
KICS_CONFIG: kics.config
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
secrets:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
app_deps_scan:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
cve_scan:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
app_sbom_analysis:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
cve_sbom_analysis:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
# Override release job to wait for stage integration tests before creating tag
|
||||
# This ensures the tag (which triggers prod deploy) is only created after stage passes
|
||||
release:
|
||||
needs: [integration_test_stage, changelog]
|
||||
|
||||
# Full integration test suite template (for feature/stage deployments)
|
||||
# Runs the complete pytest integration test suite against the deployed environment
|
||||
@@ -141,6 +206,86 @@ integration_test_stage:
|
||||
- if: '$CI_COMMIT_BRANCH == "main"'
|
||||
when: on_success
|
||||
|
||||
# Reset stage environment after integration tests (clean slate for next run)
|
||||
# Calls the /api/v1/admin/factory-reset endpoint which handles DB and S3 cleanup
|
||||
reset_stage:
|
||||
stage: deploy
|
||||
needs: [integration_test_stage]
|
||||
image: deps.global.bsf.tools/docker/python:3.12-slim
|
||||
timeout: 5m
|
||||
retry: 1 # Retry once on transient failures
|
||||
before_script:
|
||||
- pip install --index-url "$PIP_INDEX_URL" httpx
|
||||
script:
|
||||
- |
|
||||
python - <<'RESET_SCRIPT'
|
||||
import httpx
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
|
||||
BASE_URL = os.environ.get("STAGE_URL", "")
|
||||
ADMIN_USER = "admin"
|
||||
ADMIN_PASS = "changeme123" # Default admin password
|
||||
MAX_RETRIES = 3
|
||||
RETRY_DELAY = 5 # seconds
|
||||
|
||||
if not BASE_URL:
|
||||
print("ERROR: STAGE_URL environment variable not set")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"=== Resetting stage environment at {BASE_URL} ===")
|
||||
|
||||
def do_reset():
|
||||
with httpx.Client(base_url=BASE_URL, timeout=120.0) as client:
|
||||
# Login as admin
|
||||
print("Logging in as admin...")
|
||||
login_response = client.post(
|
||||
"/api/v1/auth/login",
|
||||
json={"username": ADMIN_USER, "password": ADMIN_PASS},
|
||||
)
|
||||
if login_response.status_code != 200:
|
||||
raise Exception(f"Login failed: {login_response.status_code} - {login_response.text}")
|
||||
print("Login successful")
|
||||
|
||||
# Call factory reset endpoint
|
||||
print("Calling factory reset endpoint...")
|
||||
reset_response = client.post(
|
||||
"/api/v1/admin/factory-reset",
|
||||
headers={"X-Confirm-Reset": "yes-delete-all-data"},
|
||||
)
|
||||
|
||||
if reset_response.status_code == 200:
|
||||
result = reset_response.json()
|
||||
print("Factory reset successful!")
|
||||
print(f" Database tables dropped: {result['results']['database_tables_dropped']}")
|
||||
print(f" S3 objects deleted: {result['results']['s3_objects_deleted']}")
|
||||
print(f" Database reinitialized: {result['results']['database_reinitialized']}")
|
||||
print(f" Seeded: {result['results']['seeded']}")
|
||||
return True
|
||||
else:
|
||||
raise Exception(f"Factory reset failed: {reset_response.status_code} - {reset_response.text}")
|
||||
|
||||
# Retry loop
|
||||
for attempt in range(1, MAX_RETRIES + 1):
|
||||
try:
|
||||
print(f"Attempt {attempt}/{MAX_RETRIES}")
|
||||
if do_reset():
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
print(f"Attempt {attempt} failed: {e}")
|
||||
if attempt < MAX_RETRIES:
|
||||
print(f"Retrying in {RETRY_DELAY} seconds...")
|
||||
time.sleep(RETRY_DELAY)
|
||||
else:
|
||||
print("All retry attempts failed")
|
||||
sys.exit(1)
|
||||
RESET_SCRIPT
|
||||
rules:
|
||||
- if: '$CI_COMMIT_BRANCH == "main"'
|
||||
when: on_success
|
||||
allow_failure: true # Don't fail pipeline if reset has issues
|
||||
|
||||
# Integration tests for feature deployment (full suite)
|
||||
integration_test_feature:
|
||||
<<: *integration_test_template
|
||||
@@ -183,6 +328,10 @@ python_unit_tests:
|
||||
coverage_format: cobertura
|
||||
path: backend/coverage.xml
|
||||
coverage: '/TOTAL.*\s+(\d+%)/'
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
# Run frontend tests
|
||||
frontend_tests:
|
||||
@@ -212,6 +361,10 @@ frontend_tests:
|
||||
coverage_format: cobertura
|
||||
path: frontend/coverage/cobertura-coverage.xml
|
||||
coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
# Shared deploy configuration
|
||||
.deploy_template: &deploy_template
|
||||
@@ -336,15 +489,14 @@ cleanup_feature:
|
||||
when: manual
|
||||
allow_failure: true
|
||||
|
||||
# Deploy to production (version tags only, manual approval required)
|
||||
# Deploy to production (version tags only)
|
||||
deploy_prod:
|
||||
stage: deploy
|
||||
# For tag pipelines, most jobs don't run (trusting main was tested)
|
||||
# We only need build_image to have the image available
|
||||
needs: [build_image]
|
||||
# For tag pipelines, no other jobs run - image was already built when commit was on main
|
||||
needs: []
|
||||
image: deps.global.bsf.tools/registry-1.docker.io/alpine/k8s:1.29.12
|
||||
variables:
|
||||
NAMESPACE: orch-prod-namespace
|
||||
NAMESPACE: orch-namespace
|
||||
VALUES_FILE: helm/orchard/values-prod.yaml
|
||||
BASE_URL: $PROD_URL
|
||||
before_script:
|
||||
@@ -371,7 +523,7 @@ deploy_prod:
|
||||
rules:
|
||||
# Only run on semantic version tags (v1.0.0, v1.2.3, etc.)
|
||||
- if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/'
|
||||
when: manual # Require manual approval for prod
|
||||
when: on_success
|
||||
allow_failure: false
|
||||
|
||||
# Smoke tests for production deployment (read-only, no test data creation)
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
# False positive: s3_key is an attribute name in test assertions, not a secret
|
||||
# These are historical commits - files have since been deleted or updated with inline comments
|
||||
7e68baed0886a3c928644cd01aa3b39f92d4f976:backend/tests/test_duplicate_detection.py:generic-api-key:154
|
||||
81458b3bcb5ace97109ba4c16f4afa6e55b1b8bd:backend/tests/test_duplicate_detection.py:generic-api-key:154
|
||||
2f1891cf0126ec0e7d4c789d872a2cb2dd3a1745:backend/tests/unit/test_storage.py:generic-api-key:381
|
||||
10d36947948de796f0bacea3827f4531529c405d:backend/tests/unit/test_storage.py:generic-api-key:381
|
||||
bccbc71c13570d14b8b26a11335c45f102fe3072:backend/tests/unit/test_storage.py:generic-api-key:381
|
||||
@@ -15,3 +16,4 @@ bccbc71c13570d14b8b26a11335c45f102fe3072:backend/tests/unit/test_storage.py:gene
|
||||
08dce6cbb836b687002751fed4159bfc2da61f8b:backend/tests/unit/test_storage.py:generic-api-key:381
|
||||
617bcbe89cff9a009d77e4f1f1864efed1820e63:backend/tests/unit/test_storage.py:generic-api-key:381
|
||||
1cbd33544388e0fe6db752fa8886fab33cf9ce7c:backend/tests/unit/test_storage.py:generic-api-key:381
|
||||
7cfad28f678f5a5b8b927d694a17b9ba446b7138:backend/tests/unit/test_storage.py:generic-api-key:381
|
||||
|
||||
31
CHANGELOG.md
31
CHANGELOG.md
@@ -6,7 +6,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
### Changed
|
||||
- Simplified tag pipeline to only run deploy and smoke tests (image already built on main) (#54)
|
||||
|
||||
### Fixed
|
||||
- Fixed production CI deployment namespace to use correct `orch-namespace` (#54)
|
||||
|
||||
## [0.5.0] - 2026-01-23
|
||||
### Added
|
||||
- Added factory reset endpoint `POST /api/v1/admin/factory-reset` for test environment cleanup (#54)
|
||||
- Requires admin authentication and `X-Confirm-Reset: yes-delete-all-data` header
|
||||
- Drops all database tables, clears S3 bucket, reinitializes schema, re-seeds default data
|
||||
- CI pipeline automatically calls this after integration tests on stage
|
||||
- Added `delete_all()` method to storage backend for bulk S3 object deletion (#54)
|
||||
- Added AWS Secrets Manager CSI driver support for database credentials (#54)
|
||||
- Added SecretProviderClass template for Secrets Manager integration (#54)
|
||||
- Added IRSA service account annotations for prod and stage environments (#54)
|
||||
- Added comprehensive upload/download tests for size boundaries (1B to 1GB) (#38)
|
||||
- Added concurrent upload/download tests (2, 5, 10 parallel operations) (#38)
|
||||
- Added data integrity tests (binary, text, unicode, compressed content) (#38)
|
||||
@@ -61,6 +76,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Added internal proxy configuration for npm, pip, helm, and apt (#51)
|
||||
|
||||
### Changed
|
||||
- Configured stage and prod to use AWS RDS instead of PostgreSQL subchart (#54)
|
||||
- Configured stage and prod to use AWS S3 instead of MinIO subchart (#54)
|
||||
- Changed prod deployment from manual to automatic on version tags (#54)
|
||||
- Updated S3 client to support IRSA credentials when no explicit keys provided (#54)
|
||||
- Changed prod image pullPolicy to Always (#54)
|
||||
- Added proxy-body-size annotation to prod ingress for large uploads (#54)
|
||||
- CI integration tests now run full pytest suite (~350 tests) against deployed environment instead of 3 smoke tests
|
||||
- CI production deployment uses lightweight smoke tests only (no test data creation in prod)
|
||||
- CI pipeline improvements: shared pip cache, `interruptible` flag on test jobs, retry on integration tests
|
||||
@@ -81,6 +102,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Improved pod naming: Orchard pods now named `orchard-{env}-server-*` for clarity (#51)
|
||||
|
||||
### Fixed
|
||||
- Fixed factory reset not creating default admin user after reset (#60)
|
||||
- Admin user was only created at server startup, not after factory reset
|
||||
- CI reset job would fail to login because admin user didn't exist
|
||||
- Improved reset_stage CI job reliability (#60)
|
||||
- Added application-level retry logic (3 attempts with 5s delay)
|
||||
- Added job-level retry for transient failures
|
||||
- Fixed httpx client to use proper context manager
|
||||
- Increased timeout to 120s for reset operations
|
||||
- Fixed CI integration test rate limiting: added configurable `ORCHARD_LOGIN_RATE_LIMIT` env var, relaxed to 1000/minute for dev/stage
|
||||
- Fixed duplicate `TestSecurityEdgeCases` class definition in test_auth_api.py
|
||||
- Fixed integration tests auth: session-scoped client, configurable credentials via env vars, fail-fast on auth errors
|
||||
@@ -101,6 +130,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Removed
|
||||
- Removed unused `store_streaming()` method from storage.py (#51)
|
||||
- Disabled PostgreSQL subchart for stage and prod environments (#54)
|
||||
- Disabled MinIO subchart for stage and prod environments (#54)
|
||||
|
||||
## [0.4.0] - 2026-01-12
|
||||
### Added
|
||||
|
||||
@@ -6390,3 +6390,121 @@ def get_artifact_provenance(
|
||||
tags=tag_list,
|
||||
uploads=upload_history,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Factory Reset Endpoint (Admin Only)
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@router.post("/api/v1/admin/factory-reset", tags=["admin"])
|
||||
def factory_reset(
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
storage: S3Storage = Depends(get_storage),
|
||||
current_user: User = Depends(require_admin),
|
||||
):
|
||||
"""
|
||||
Factory reset - delete all data and restore to initial state.
|
||||
|
||||
This endpoint:
|
||||
1. Drops all database tables
|
||||
2. Deletes all objects from S3 storage
|
||||
3. Recreates the database schema
|
||||
4. Re-seeds with default admin user
|
||||
|
||||
Requires:
|
||||
- Admin authentication
|
||||
- X-Confirm-Reset header set to "yes-delete-all-data"
|
||||
|
||||
WARNING: This is a destructive operation that cannot be undone.
|
||||
"""
|
||||
# Require explicit confirmation header
|
||||
confirm_header = request.headers.get("X-Confirm-Reset")
|
||||
if confirm_header != "yes-delete-all-data":
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="Factory reset requires X-Confirm-Reset header set to 'yes-delete-all-data'",
|
||||
)
|
||||
|
||||
# Capture username before we drop tables (user object will become invalid)
|
||||
admin_username = current_user.username
|
||||
logger.warning(f"Factory reset initiated by admin user: {admin_username}")
|
||||
|
||||
results = {
|
||||
"database_tables_dropped": 0,
|
||||
"s3_objects_deleted": 0,
|
||||
"database_reinitialized": False,
|
||||
"seeded": False,
|
||||
}
|
||||
|
||||
try:
|
||||
# Step 1: Drop all tables in public schema
|
||||
logger.info("Dropping all database tables...")
|
||||
drop_result = db.execute(
|
||||
text("""
|
||||
DO $$
|
||||
DECLARE
|
||||
r RECORD;
|
||||
table_count INT := 0;
|
||||
BEGIN
|
||||
SET session_replication_role = 'replica';
|
||||
FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = 'public') LOOP
|
||||
EXECUTE 'DROP TABLE IF EXISTS public.' || quote_ident(r.tablename) || ' CASCADE';
|
||||
table_count := table_count + 1;
|
||||
END LOOP;
|
||||
SET session_replication_role = 'origin';
|
||||
RAISE NOTICE 'Dropped % tables', table_count;
|
||||
END $$;
|
||||
""")
|
||||
)
|
||||
db.commit()
|
||||
|
||||
# Count tables that were dropped
|
||||
count_result = db.execute(
|
||||
text("SELECT COUNT(*) FROM pg_tables WHERE schemaname = 'public'")
|
||||
)
|
||||
remaining_tables = count_result.scalar()
|
||||
results["database_tables_dropped"] = "all"
|
||||
logger.info(f"Database tables dropped, remaining: {remaining_tables}")
|
||||
|
||||
# Step 2: Delete all S3 objects
|
||||
logger.info("Deleting all S3 objects...")
|
||||
results["s3_objects_deleted"] = storage.delete_all()
|
||||
|
||||
# Step 3: Reinitialize database schema
|
||||
logger.info("Reinitializing database schema...")
|
||||
from .database import init_db, SessionLocal
|
||||
init_db()
|
||||
results["database_reinitialized"] = True
|
||||
|
||||
# Step 4: Re-seed with default data (need fresh session after schema recreate)
|
||||
logger.info("Seeding database with defaults...")
|
||||
from .seed import seed_database
|
||||
from .auth import create_default_admin
|
||||
fresh_db = SessionLocal()
|
||||
try:
|
||||
# Create default admin user first (normally done at startup)
|
||||
create_default_admin(fresh_db)
|
||||
# Then seed other test data
|
||||
seed_database(fresh_db)
|
||||
fresh_db.commit()
|
||||
finally:
|
||||
fresh_db.close()
|
||||
results["seeded"] = True
|
||||
|
||||
logger.warning(f"Factory reset completed by {admin_username}")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Factory reset completed successfully",
|
||||
"results": results,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Factory reset failed: {e}")
|
||||
db.rollback()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Factory reset failed: {str(e)}",
|
||||
)
|
||||
|
||||
@@ -242,15 +242,19 @@ class S3Storage:
|
||||
},
|
||||
)
|
||||
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.s3_endpoint if settings.s3_endpoint else None,
|
||||
region_name=settings.s3_region,
|
||||
aws_access_key_id=settings.s3_access_key_id,
|
||||
aws_secret_access_key=settings.s3_secret_access_key,
|
||||
config=config,
|
||||
verify=settings.s3_verify_ssl, # SSL/TLS verification
|
||||
)
|
||||
# Build client kwargs - only include credentials if explicitly provided
|
||||
# This allows IRSA/IAM role credentials to be used when no explicit creds are set
|
||||
client_kwargs = {
|
||||
"endpoint_url": settings.s3_endpoint if settings.s3_endpoint else None,
|
||||
"region_name": settings.s3_region,
|
||||
"config": config,
|
||||
"verify": settings.s3_verify_ssl,
|
||||
}
|
||||
if settings.s3_access_key_id and settings.s3_secret_access_key:
|
||||
client_kwargs["aws_access_key_id"] = settings.s3_access_key_id
|
||||
client_kwargs["aws_secret_access_key"] = settings.s3_secret_access_key
|
||||
|
||||
self.client = boto3.client("s3", **client_kwargs)
|
||||
self.bucket = settings.s3_bucket
|
||||
# Store active multipart uploads for resumable support
|
||||
self._active_uploads: Dict[str, Dict[str, Any]] = {}
|
||||
@@ -831,6 +835,36 @@ class S3Storage:
|
||||
except ClientError:
|
||||
return False
|
||||
|
||||
def delete_all(self) -> int:
|
||||
"""
|
||||
Delete all objects in the bucket.
|
||||
|
||||
Returns:
|
||||
Number of objects deleted
|
||||
"""
|
||||
deleted_count = 0
|
||||
try:
|
||||
paginator = self.client.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(Bucket=self.bucket):
|
||||
objects = page.get("Contents", [])
|
||||
if not objects:
|
||||
continue
|
||||
|
||||
# Delete objects in batches of 1000 (S3 limit)
|
||||
delete_keys = [{"Key": obj["Key"]} for obj in objects]
|
||||
if delete_keys:
|
||||
self.client.delete_objects(
|
||||
Bucket=self.bucket, Delete={"Objects": delete_keys}
|
||||
)
|
||||
deleted_count += len(delete_keys)
|
||||
logger.info(f"Deleted {len(delete_keys)} objects from S3")
|
||||
|
||||
logger.info(f"Total objects deleted from S3: {deleted_count}")
|
||||
return deleted_count
|
||||
except ClientError as e:
|
||||
logger.error(f"Failed to delete all S3 objects: {e}")
|
||||
raise
|
||||
|
||||
def generate_presigned_url(
|
||||
self,
|
||||
s3_key: str,
|
||||
|
||||
@@ -77,6 +77,8 @@ PostgreSQL secret name
|
||||
{{- define "orchard.postgresql.secretName" -}}
|
||||
{{- if .Values.orchard.database.existingSecret }}
|
||||
{{- .Values.orchard.database.existingSecret }}
|
||||
{{- else if and .Values.orchard.database.secretsManager .Values.orchard.database.secretsManager.enabled }}
|
||||
{{- printf "%s-db-credentials" (include "orchard.fullname" .) }}
|
||||
{{- else if .Values.postgresql.enabled }}
|
||||
{{- printf "%s-postgresql" .Release.Name }}
|
||||
{{- else }}
|
||||
@@ -90,6 +92,8 @@ PostgreSQL password key in secret
|
||||
{{- define "orchard.postgresql.passwordKey" -}}
|
||||
{{- if .Values.orchard.database.existingSecret -}}
|
||||
{{- .Values.orchard.database.existingSecretPasswordKey -}}
|
||||
{{- else if and .Values.orchard.database.secretsManager .Values.orchard.database.secretsManager.enabled -}}
|
||||
password
|
||||
{{- else if .Values.postgresql.enabled -}}
|
||||
password
|
||||
{{- else -}}
|
||||
|
||||
@@ -77,8 +77,16 @@ spec:
|
||||
value: {{ include "orchard.postgresql.host" . | quote }}
|
||||
- name: ORCHARD_DATABASE_PORT
|
||||
value: {{ .Values.orchard.database.port | quote }}
|
||||
{{- if and .Values.orchard.database.secretsManager .Values.orchard.database.secretsManager.enabled }}
|
||||
- name: ORCHARD_DATABASE_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ include "orchard.postgresql.secretName" . }}
|
||||
key: username
|
||||
{{- else }}
|
||||
- name: ORCHARD_DATABASE_USER
|
||||
value: {{ .Values.orchard.database.user | default .Values.postgresql.auth.username | quote }}
|
||||
{{- end }}
|
||||
- name: ORCHARD_DATABASE_DBNAME
|
||||
value: {{ .Values.orchard.database.dbname | default .Values.postgresql.auth.database | quote }}
|
||||
- name: ORCHARD_DATABASE_SSLMODE
|
||||
@@ -96,6 +104,7 @@ spec:
|
||||
value: {{ .Values.orchard.s3.bucket | quote }}
|
||||
- name: ORCHARD_S3_USE_PATH_STYLE
|
||||
value: {{ .Values.orchard.s3.usePathStyle | quote }}
|
||||
{{- if or .Values.minio.enabled .Values.orchard.s3.existingSecret .Values.orchard.s3.accessKeyId }}
|
||||
- name: ORCHARD_S3_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
@@ -106,6 +115,7 @@ spec:
|
||||
secretKeyRef:
|
||||
name: {{ include "orchard.minio.secretName" . }}
|
||||
key: {{ if .Values.minio.enabled }}root-password{{ else }}{{ .Values.orchard.s3.existingSecretSecretKeyKey }}{{ end }}
|
||||
{{- end }}
|
||||
- name: ORCHARD_DOWNLOAD_MODE
|
||||
value: {{ .Values.orchard.download.mode | quote }}
|
||||
- name: ORCHARD_PRESIGNED_URL_EXPIRY
|
||||
@@ -116,12 +126,27 @@ spec:
|
||||
value: {{ .Values.orchard.rateLimit.login | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if and .Values.orchard.database.secretsManager .Values.orchard.database.secretsManager.enabled }}
|
||||
volumeMounts:
|
||||
- name: db-secrets
|
||||
mountPath: /mnt/secrets-store
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
livenessProbe:
|
||||
{{- toYaml .Values.livenessProbe | nindent 12 }}
|
||||
readinessProbe:
|
||||
{{- toYaml .Values.readinessProbe | nindent 12 }}
|
||||
resources:
|
||||
{{- toYaml .Values.resources | nindent 12 }}
|
||||
{{- if and .Values.orchard.database.secretsManager .Values.orchard.database.secretsManager.enabled }}
|
||||
volumes:
|
||||
- name: db-secrets
|
||||
csi:
|
||||
driver: secrets-store.csi.k8s.io
|
||||
readOnly: true
|
||||
volumeAttributes:
|
||||
secretProviderClass: {{ include "orchard.fullname" . }}-db-secret
|
||||
{{- end }}
|
||||
{{- with .Values.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
|
||||
27
helm/orchard/templates/secret-provider-class.yaml
Normal file
27
helm/orchard/templates/secret-provider-class.yaml
Normal file
@@ -0,0 +1,27 @@
|
||||
{{- if and .Values.orchard.database.secretsManager .Values.orchard.database.secretsManager.enabled }}
|
||||
apiVersion: secrets-store.csi.x-k8s.io/v1
|
||||
kind: SecretProviderClass
|
||||
metadata:
|
||||
name: {{ include "orchard.fullname" . }}-db-secret
|
||||
labels:
|
||||
{{- include "orchard.labels" . | nindent 4 }}
|
||||
spec:
|
||||
provider: aws
|
||||
parameters:
|
||||
objects: |
|
||||
- objectName: "{{ .Values.orchard.database.secretsManager.secretArn }}"
|
||||
objectType: "secretsmanager"
|
||||
jmesPath:
|
||||
- path: username
|
||||
objectAlias: db-username
|
||||
- path: password
|
||||
objectAlias: db-password
|
||||
secretObjects:
|
||||
- secretName: {{ include "orchard.fullname" . }}-db-credentials
|
||||
type: Opaque
|
||||
data:
|
||||
- objectName: db-username
|
||||
key: username
|
||||
- objectName: db-password
|
||||
key: password
|
||||
{{- end }}
|
||||
@@ -4,7 +4,7 @@ replicaCount: 1
|
||||
|
||||
image:
|
||||
repository: registry.global.bsf.tools/esv/bsf/bsf-integration/orchard/orchard-mvp
|
||||
pullPolicy: IfNotPresent # Don't always pull in prod
|
||||
pullPolicy: Always
|
||||
tag: "latest" # Overridden by CI
|
||||
|
||||
imagePullSecrets:
|
||||
@@ -19,7 +19,8 @@ initContainer:
|
||||
serviceAccount:
|
||||
create: true
|
||||
automount: true
|
||||
annotations: {}
|
||||
annotations:
|
||||
eks.amazonaws.com/role-arn: arn:aws-us-gov:iam::052673043337:role/service-orchard-prd
|
||||
name: "orchard"
|
||||
|
||||
podAnnotations: {}
|
||||
@@ -41,6 +42,7 @@ ingress:
|
||||
className: "nginx"
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: "letsencrypt"
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "0" # Disable body size limit for uploads
|
||||
hosts:
|
||||
- host: orchard.common.global.bsf.tools
|
||||
paths:
|
||||
@@ -90,122 +92,38 @@ orchard:
|
||||
host: "0.0.0.0"
|
||||
port: 8080
|
||||
|
||||
# Database configuration (used when postgresql.enabled is false)
|
||||
# TODO: Configure for managed PostgreSQL when ready
|
||||
# Database configuration - uses AWS Secrets Manager via CSI driver
|
||||
database:
|
||||
host: ""
|
||||
host: "orchard-prd.cluster-cvw3jzjkozoc.us-gov-west-1.rds.amazonaws.com"
|
||||
port: 5432
|
||||
user: orchard
|
||||
password: ""
|
||||
dbname: orchard
|
||||
sslmode: disable
|
||||
existingSecret: ""
|
||||
existingSecretPasswordKey: "password"
|
||||
dbname: orchard_prod
|
||||
sslmode: require
|
||||
secretsManager:
|
||||
enabled: true
|
||||
secretArn: "arn:aws-us-gov:secretsmanager:us-gov-west-1:052673043337:secret:rds!cluster-0afc8af5-f644-4284-92fb-2ed545490f92-3v9uXV"
|
||||
|
||||
# S3 configuration (used when minio.enabled is false)
|
||||
# TODO: Configure for real S3 when ready
|
||||
# S3 configuration - uses IRSA for credentials
|
||||
s3:
|
||||
endpoint: ""
|
||||
region: us-east-1
|
||||
bucket: orchard-artifacts
|
||||
accessKeyId: ""
|
||||
secretAccessKey: ""
|
||||
usePathStyle: true
|
||||
existingSecret: ""
|
||||
existingSecretAccessKeyKey: "access-key-id"
|
||||
existingSecretSecretKeyKey: "secret-access-key"
|
||||
endpoint: "" # Empty = use AWS default
|
||||
region: us-gov-west-1
|
||||
bucket: orchard-artifacts-prod
|
||||
usePathStyle: false # Real S3 uses virtual-hosted style
|
||||
|
||||
download:
|
||||
mode: "presigned"
|
||||
presignedUrlExpiry: 3600
|
||||
|
||||
# PostgreSQL subchart - MVP uses subchart, switch to managed later
|
||||
# PostgreSQL subchart - disabled in prod, using RDS
|
||||
postgresql:
|
||||
enabled: true
|
||||
image:
|
||||
registry: containers.global.bsf.tools
|
||||
repository: bitnami/postgresql
|
||||
tag: "15"
|
||||
pullPolicy: IfNotPresent
|
||||
auth:
|
||||
username: orchard
|
||||
password: orchard-prod-password # TODO: Use existingSecret
|
||||
database: orchard
|
||||
primary:
|
||||
persistence:
|
||||
enabled: true # Enable persistence for prod
|
||||
size: 20Gi
|
||||
resourcesPreset: "none"
|
||||
resources:
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
volumePermissions:
|
||||
resourcesPreset: "none"
|
||||
resources:
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 128Mi
|
||||
enabled: false
|
||||
|
||||
# MinIO subchart - MVP uses subchart, switch to real S3 later
|
||||
# MinIO subchart - disabled in prod, using real S3
|
||||
minio:
|
||||
enabled: true
|
||||
image:
|
||||
registry: containers.global.bsf.tools
|
||||
repository: bitnami/minio
|
||||
tag: "latest"
|
||||
pullPolicy: IfNotPresent
|
||||
auth:
|
||||
rootUser: minioadmin
|
||||
rootPassword: minioadmin-prod # TODO: Use existingSecret
|
||||
defaultBuckets: "orchard-artifacts"
|
||||
persistence:
|
||||
enabled: true # Enable persistence for prod
|
||||
size: 100Gi
|
||||
resourcesPreset: "none"
|
||||
resources:
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
defaultInitContainers:
|
||||
volumePermissions:
|
||||
resourcesPreset: "none"
|
||||
resources:
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 128Mi
|
||||
provisioning:
|
||||
resources:
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
enabled: false
|
||||
|
||||
# MinIO external ingress for presigned URL access
|
||||
# MinIO ingress - disabled in prod, using real S3
|
||||
minioIngress:
|
||||
enabled: true
|
||||
className: "nginx"
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: "letsencrypt"
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "0"
|
||||
host: "minio-orchard.common.global.bsf.tools"
|
||||
tls:
|
||||
enabled: true
|
||||
secretName: minio-prod-tls
|
||||
enabled: false
|
||||
|
||||
redis:
|
||||
enabled: false
|
||||
|
||||
@@ -19,7 +19,8 @@ initContainer:
|
||||
serviceAccount:
|
||||
create: true
|
||||
automount: true
|
||||
annotations: {}
|
||||
annotations:
|
||||
eks.amazonaws.com/role-arn: arn:aws-us-gov:iam::052673043337:role/service-orchard-stage
|
||||
name: "orchard"
|
||||
|
||||
podAnnotations: {}
|
||||
@@ -93,28 +94,22 @@ orchard:
|
||||
host: "0.0.0.0"
|
||||
port: 8080
|
||||
|
||||
# Database configuration (used when postgresql.enabled is false)
|
||||
# Database configuration - uses AWS Secrets Manager via CSI driver
|
||||
database:
|
||||
host: ""
|
||||
host: "orchard-stage.cluster-cvw3jzjkozoc.us-gov-west-1.rds.amazonaws.com"
|
||||
port: 5432
|
||||
user: orchard
|
||||
password: ""
|
||||
dbname: orchard
|
||||
sslmode: disable
|
||||
existingSecret: ""
|
||||
existingSecretPasswordKey: "password"
|
||||
dbname: postgres
|
||||
sslmode: require
|
||||
secretsManager:
|
||||
enabled: true
|
||||
secretArn: "arn:aws-us-gov:secretsmanager:us-gov-west-1:052673043337:secret:rds!cluster-a573672b-1a38-4665-a654-1b7df37b5297-IaeFQL"
|
||||
|
||||
# S3 configuration (used when minio.enabled is false)
|
||||
# S3 configuration - uses IRSA for credentials
|
||||
s3:
|
||||
endpoint: ""
|
||||
region: us-east-1
|
||||
bucket: orchard-artifacts
|
||||
accessKeyId: ""
|
||||
secretAccessKey: ""
|
||||
usePathStyle: true
|
||||
existingSecret: ""
|
||||
existingSecretAccessKeyKey: "access-key-id"
|
||||
existingSecretSecretKeyKey: "secret-access-key"
|
||||
endpoint: "" # Empty = use AWS default
|
||||
region: us-gov-west-1
|
||||
bucket: orchard-artifacts-stage
|
||||
usePathStyle: false # Real S3 uses virtual-hosted style
|
||||
|
||||
# Download configuration
|
||||
download:
|
||||
@@ -125,98 +120,17 @@ orchard:
|
||||
rateLimit:
|
||||
login: "1000/minute" # Default is 5/minute, relaxed for CI integration tests
|
||||
|
||||
# PostgreSQL subchart configuration
|
||||
# PostgreSQL subchart - disabled in stage, using RDS
|
||||
postgresql:
|
||||
enabled: true
|
||||
image:
|
||||
registry: containers.global.bsf.tools
|
||||
repository: bitnami/postgresql
|
||||
tag: "15"
|
||||
pullPolicy: IfNotPresent
|
||||
auth:
|
||||
username: orchard
|
||||
password: orchard-password
|
||||
database: orchard
|
||||
primary:
|
||||
persistence:
|
||||
enabled: false
|
||||
size: 10Gi
|
||||
# Resources with memory requests = limits per cluster policy
|
||||
resourcesPreset: "none"
|
||||
resources:
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
# Volume permissions init container
|
||||
volumePermissions:
|
||||
resourcesPreset: "none"
|
||||
resources:
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 128Mi
|
||||
enabled: false
|
||||
|
||||
# MinIO subchart configuration
|
||||
# MinIO subchart - disabled in stage, using real S3
|
||||
minio:
|
||||
enabled: true
|
||||
image:
|
||||
registry: containers.global.bsf.tools
|
||||
repository: bitnami/minio
|
||||
tag: "latest"
|
||||
pullPolicy: IfNotPresent
|
||||
auth:
|
||||
rootUser: minioadmin
|
||||
rootPassword: minioadmin
|
||||
defaultBuckets: "orchard-artifacts"
|
||||
persistence:
|
||||
enabled: false
|
||||
size: 50Gi
|
||||
# Resources with memory requests = limits per cluster policy
|
||||
resourcesPreset: "none" # Disable preset to use explicit resources
|
||||
resources:
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
# Init container resources
|
||||
defaultInitContainers:
|
||||
volumePermissions:
|
||||
resourcesPreset: "none"
|
||||
resources:
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 128Mi
|
||||
# Provisioning job resources
|
||||
provisioning:
|
||||
resources:
|
||||
limits:
|
||||
cpu: 200m
|
||||
memory: 256Mi
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
enabled: false
|
||||
|
||||
# MinIO external ingress for presigned URL access (separate from subchart ingress)
|
||||
# MinIO ingress - disabled in stage, using real S3
|
||||
minioIngress:
|
||||
enabled: true
|
||||
className: "nginx"
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: "letsencrypt"
|
||||
nginx.ingress.kubernetes.io/proxy-body-size: "0" # Disable body size limit for uploads
|
||||
host: "minio-orch-stage.common.global.bsf.tools"
|
||||
tls:
|
||||
enabled: true
|
||||
secretName: minio-tls
|
||||
enabled: false
|
||||
|
||||
# Redis subchart configuration (for future caching)
|
||||
redis:
|
||||
|
||||
Reference in New Issue
Block a user