Compare commits
12 Commits
feature/st
...
fix/ci-pro
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0e01e17dfe | ||
|
|
ca46ab1ea0 | ||
|
|
a01c45cb64 | ||
|
|
133d9cbfd6 | ||
|
|
276b4f2743 | ||
|
|
67ac6bb3f8 | ||
|
|
b0bb3ed569 | ||
|
|
1ac75e1017 | ||
|
|
693613f111 | ||
|
|
9da4ae8c0d | ||
|
|
7ffdc64364 | ||
|
|
6abc0c88b0 |
147
.gitlab-ci.yml
147
.gitlab-ci.yml
@@ -36,9 +36,68 @@ stages:
|
||||
- analyze
|
||||
- deploy
|
||||
|
||||
# Override Prosper template jobs to exclude tag pipelines
|
||||
# Tags only run deploy_prod and smoke_test_prod (image already built on main)
|
||||
build_image:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
test_image:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
hadolint:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
kics:
|
||||
variables:
|
||||
KICS_CONFIG: kics.config
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
secrets:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
app_deps_scan:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
cve_scan:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
app_sbom_analysis:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
cve_sbom_analysis:
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
# Override release job to wait for stage integration tests before creating tag
|
||||
# This ensures the tag (which triggers prod deploy) is only created after stage passes
|
||||
release:
|
||||
needs: [integration_test_stage, changelog]
|
||||
|
||||
# Full integration test suite template (for feature/stage deployments)
|
||||
# Runs the complete pytest integration test suite against the deployed environment
|
||||
@@ -154,6 +213,7 @@ reset_stage:
|
||||
needs: [integration_test_stage]
|
||||
image: deps.global.bsf.tools/docker/python:3.12-slim
|
||||
timeout: 5m
|
||||
retry: 1 # Retry once on transient failures
|
||||
before_script:
|
||||
- pip install --index-url "$PIP_INDEX_URL" httpx
|
||||
script:
|
||||
@@ -162,10 +222,13 @@ reset_stage:
|
||||
import httpx
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
|
||||
BASE_URL = os.environ.get("STAGE_URL", "")
|
||||
ADMIN_USER = "admin"
|
||||
ADMIN_PASS = "changeme123" # Default admin password
|
||||
MAX_RETRIES = 3
|
||||
RETRY_DELAY = 5 # seconds
|
||||
|
||||
if not BASE_URL:
|
||||
print("ERROR: STAGE_URL environment variable not set")
|
||||
@@ -173,37 +236,50 @@ reset_stage:
|
||||
|
||||
print(f"=== Resetting stage environment at {BASE_URL} ===")
|
||||
|
||||
client = httpx.Client(base_url=BASE_URL, timeout=60.0)
|
||||
def do_reset():
|
||||
with httpx.Client(base_url=BASE_URL, timeout=120.0) as client:
|
||||
# Login as admin
|
||||
print("Logging in as admin...")
|
||||
login_response = client.post(
|
||||
"/api/v1/auth/login",
|
||||
json={"username": ADMIN_USER, "password": ADMIN_PASS},
|
||||
)
|
||||
if login_response.status_code != 200:
|
||||
raise Exception(f"Login failed: {login_response.status_code} - {login_response.text}")
|
||||
print("Login successful")
|
||||
|
||||
# Login as admin
|
||||
print("Logging in as admin...")
|
||||
login_response = client.post(
|
||||
"/api/v1/auth/login",
|
||||
json={"username": ADMIN_USER, "password": ADMIN_PASS},
|
||||
)
|
||||
if login_response.status_code != 200:
|
||||
print(f"Login failed: {login_response.status_code} - {login_response.text}")
|
||||
sys.exit(1)
|
||||
print("Login successful")
|
||||
# Call factory reset endpoint
|
||||
print("Calling factory reset endpoint...")
|
||||
reset_response = client.post(
|
||||
"/api/v1/admin/factory-reset",
|
||||
headers={"X-Confirm-Reset": "yes-delete-all-data"},
|
||||
)
|
||||
|
||||
# Call factory reset endpoint
|
||||
print("Calling factory reset endpoint...")
|
||||
reset_response = client.post(
|
||||
"/api/v1/admin/factory-reset",
|
||||
headers={"X-Confirm-Reset": "yes-delete-all-data"},
|
||||
)
|
||||
if reset_response.status_code == 200:
|
||||
result = reset_response.json()
|
||||
print("Factory reset successful!")
|
||||
print(f" Database tables dropped: {result['results']['database_tables_dropped']}")
|
||||
print(f" S3 objects deleted: {result['results']['s3_objects_deleted']}")
|
||||
print(f" Database reinitialized: {result['results']['database_reinitialized']}")
|
||||
print(f" Seeded: {result['results']['seeded']}")
|
||||
return True
|
||||
else:
|
||||
raise Exception(f"Factory reset failed: {reset_response.status_code} - {reset_response.text}")
|
||||
|
||||
if reset_response.status_code == 200:
|
||||
result = reset_response.json()
|
||||
print(f"Factory reset successful!")
|
||||
print(f" Database tables dropped: {result['results']['database_tables_dropped']}")
|
||||
print(f" S3 objects deleted: {result['results']['s3_objects_deleted']}")
|
||||
print(f" Database reinitialized: {result['results']['database_reinitialized']}")
|
||||
print(f" Seeded: {result['results']['seeded']}")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print(f"Factory reset failed: {reset_response.status_code} - {reset_response.text}")
|
||||
sys.exit(1)
|
||||
# Retry loop
|
||||
for attempt in range(1, MAX_RETRIES + 1):
|
||||
try:
|
||||
print(f"Attempt {attempt}/{MAX_RETRIES}")
|
||||
if do_reset():
|
||||
sys.exit(0)
|
||||
except Exception as e:
|
||||
print(f"Attempt {attempt} failed: {e}")
|
||||
if attempt < MAX_RETRIES:
|
||||
print(f"Retrying in {RETRY_DELAY} seconds...")
|
||||
time.sleep(RETRY_DELAY)
|
||||
else:
|
||||
print("All retry attempts failed")
|
||||
sys.exit(1)
|
||||
RESET_SCRIPT
|
||||
rules:
|
||||
- if: '$CI_COMMIT_BRANCH == "main"'
|
||||
@@ -252,6 +328,10 @@ python_unit_tests:
|
||||
coverage_format: cobertura
|
||||
path: backend/coverage.xml
|
||||
coverage: '/TOTAL.*\s+(\d+%)/'
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
# Run frontend tests
|
||||
frontend_tests:
|
||||
@@ -281,6 +361,10 @@ frontend_tests:
|
||||
coverage_format: cobertura
|
||||
path: frontend/coverage/cobertura-coverage.xml
|
||||
coverage: '/All files[^|]*\|[^|]*\s+([\d\.]+)/'
|
||||
rules:
|
||||
- if: '$CI_COMMIT_TAG'
|
||||
when: never
|
||||
- when: on_success
|
||||
|
||||
# Shared deploy configuration
|
||||
.deploy_template: &deploy_template
|
||||
@@ -408,12 +492,11 @@ cleanup_feature:
|
||||
# Deploy to production (version tags only)
|
||||
deploy_prod:
|
||||
stage: deploy
|
||||
# For tag pipelines, most jobs don't run (trusting main was tested)
|
||||
# We only need build_image to have the image available
|
||||
needs: [build_image]
|
||||
# For tag pipelines, no other jobs run - image was already built when commit was on main
|
||||
needs: []
|
||||
image: deps.global.bsf.tools/registry-1.docker.io/alpine/k8s:1.29.12
|
||||
variables:
|
||||
NAMESPACE: orch-prod-namespace
|
||||
NAMESPACE: orch-namespace
|
||||
VALUES_FILE: helm/orchard/values-prod.yaml
|
||||
BASE_URL: $PROD_URL
|
||||
before_script:
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
# False positive: s3_key is an attribute name in test assertions, not a secret
|
||||
# These are historical commits - files have since been deleted or updated with inline comments
|
||||
7e68baed0886a3c928644cd01aa3b39f92d4f976:backend/tests/test_duplicate_detection.py:generic-api-key:154
|
||||
81458b3bcb5ace97109ba4c16f4afa6e55b1b8bd:backend/tests/test_duplicate_detection.py:generic-api-key:154
|
||||
2f1891cf0126ec0e7d4c789d872a2cb2dd3a1745:backend/tests/unit/test_storage.py:generic-api-key:381
|
||||
10d36947948de796f0bacea3827f4531529c405d:backend/tests/unit/test_storage.py:generic-api-key:381
|
||||
bccbc71c13570d14b8b26a11335c45f102fe3072:backend/tests/unit/test_storage.py:generic-api-key:381
|
||||
@@ -15,3 +16,4 @@ bccbc71c13570d14b8b26a11335c45f102fe3072:backend/tests/unit/test_storage.py:gene
|
||||
08dce6cbb836b687002751fed4159bfc2da61f8b:backend/tests/unit/test_storage.py:generic-api-key:381
|
||||
617bcbe89cff9a009d77e4f1f1864efed1820e63:backend/tests/unit/test_storage.py:generic-api-key:381
|
||||
1cbd33544388e0fe6db752fa8886fab33cf9ce7c:backend/tests/unit/test_storage.py:generic-api-key:381
|
||||
7cfad28f678f5a5b8b927d694a17b9ba446b7138:backend/tests/unit/test_storage.py:generic-api-key:381
|
||||
|
||||
37
CHANGELOG.md
37
CHANGELOG.md
@@ -6,6 +6,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [Unreleased]
|
||||
### Changed
|
||||
- Simplified tag pipeline to only run deploy and smoke tests (image already built on main) (#54)
|
||||
|
||||
### Fixed
|
||||
- Fixed production CI deployment namespace to use correct `orch-namespace` (#54)
|
||||
|
||||
## [0.5.0] - 2026-01-23
|
||||
### Added
|
||||
- Added factory reset endpoint `POST /api/v1/admin/factory-reset` for test environment cleanup (#54)
|
||||
- Requires admin authentication and `X-Confirm-Reset: yes-delete-all-data` header
|
||||
@@ -15,20 +22,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Added AWS Secrets Manager CSI driver support for database credentials (#54)
|
||||
- Added SecretProviderClass template for Secrets Manager integration (#54)
|
||||
- Added IRSA service account annotations for prod and stage environments (#54)
|
||||
|
||||
### Changed
|
||||
- Configured stage and prod to use AWS RDS instead of PostgreSQL subchart (#54)
|
||||
- Configured stage and prod to use AWS S3 instead of MinIO subchart (#54)
|
||||
- Changed prod deployment from manual to automatic on version tags (#54)
|
||||
- Updated S3 client to support IRSA credentials when no explicit keys provided (#54)
|
||||
- Changed prod image pullPolicy to Always (#54)
|
||||
- Added proxy-body-size annotation to prod ingress for large uploads (#54)
|
||||
|
||||
### Removed
|
||||
- Disabled PostgreSQL subchart for stage and prod environments (#54)
|
||||
- Disabled MinIO subchart for stage and prod environments (#54)
|
||||
|
||||
### Added
|
||||
- Added comprehensive upload/download tests for size boundaries (1B to 1GB) (#38)
|
||||
- Added concurrent upload/download tests (2, 5, 10 parallel operations) (#38)
|
||||
- Added data integrity tests (binary, text, unicode, compressed content) (#38)
|
||||
@@ -83,6 +76,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Added internal proxy configuration for npm, pip, helm, and apt (#51)
|
||||
|
||||
### Changed
|
||||
- Configured stage and prod to use AWS RDS instead of PostgreSQL subchart (#54)
|
||||
- Configured stage and prod to use AWS S3 instead of MinIO subchart (#54)
|
||||
- Changed prod deployment from manual to automatic on version tags (#54)
|
||||
- Updated S3 client to support IRSA credentials when no explicit keys provided (#54)
|
||||
- Changed prod image pullPolicy to Always (#54)
|
||||
- Added proxy-body-size annotation to prod ingress for large uploads (#54)
|
||||
- CI integration tests now run full pytest suite (~350 tests) against deployed environment instead of 3 smoke tests
|
||||
- CI production deployment uses lightweight smoke tests only (no test data creation in prod)
|
||||
- CI pipeline improvements: shared pip cache, `interruptible` flag on test jobs, retry on integration tests
|
||||
@@ -103,6 +102,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Improved pod naming: Orchard pods now named `orchard-{env}-server-*` for clarity (#51)
|
||||
|
||||
### Fixed
|
||||
- Fixed factory reset not creating default admin user after reset (#60)
|
||||
- Admin user was only created at server startup, not after factory reset
|
||||
- CI reset job would fail to login because admin user didn't exist
|
||||
- Improved reset_stage CI job reliability (#60)
|
||||
- Added application-level retry logic (3 attempts with 5s delay)
|
||||
- Added job-level retry for transient failures
|
||||
- Fixed httpx client to use proper context manager
|
||||
- Increased timeout to 120s for reset operations
|
||||
- Fixed CI integration test rate limiting: added configurable `ORCHARD_LOGIN_RATE_LIMIT` env var, relaxed to 1000/minute for dev/stage
|
||||
- Fixed duplicate `TestSecurityEdgeCases` class definition in test_auth_api.py
|
||||
- Fixed integration tests auth: session-scoped client, configurable credentials via env vars, fail-fast on auth errors
|
||||
@@ -123,6 +130,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Removed
|
||||
- Removed unused `store_streaming()` method from storage.py (#51)
|
||||
- Disabled PostgreSQL subchart for stage and prod environments (#54)
|
||||
- Disabled MinIO subchart for stage and prod environments (#54)
|
||||
|
||||
## [0.4.0] - 2026-01-12
|
||||
### Added
|
||||
|
||||
@@ -6427,7 +6427,9 @@ def factory_reset(
|
||||
detail="Factory reset requires X-Confirm-Reset header set to 'yes-delete-all-data'",
|
||||
)
|
||||
|
||||
logger.warning(f"Factory reset initiated by admin user: {current_user.username}")
|
||||
# Capture username before we drop tables (user object will become invalid)
|
||||
admin_username = current_user.username
|
||||
logger.warning(f"Factory reset initiated by admin user: {admin_username}")
|
||||
|
||||
results = {
|
||||
"database_tables_dropped": 0,
|
||||
@@ -6472,17 +6474,26 @@ def factory_reset(
|
||||
|
||||
# Step 3: Reinitialize database schema
|
||||
logger.info("Reinitializing database schema...")
|
||||
from .database import init_db
|
||||
from .database import init_db, SessionLocal
|
||||
init_db()
|
||||
results["database_reinitialized"] = True
|
||||
|
||||
# Step 4: Re-seed with default data
|
||||
# Step 4: Re-seed with default data (need fresh session after schema recreate)
|
||||
logger.info("Seeding database with defaults...")
|
||||
from .seed import seed_database
|
||||
seed_database()
|
||||
from .auth import create_default_admin
|
||||
fresh_db = SessionLocal()
|
||||
try:
|
||||
# Create default admin user first (normally done at startup)
|
||||
create_default_admin(fresh_db)
|
||||
# Then seed other test data
|
||||
seed_database(fresh_db)
|
||||
fresh_db.commit()
|
||||
finally:
|
||||
fresh_db.close()
|
||||
results["seeded"] = True
|
||||
|
||||
logger.warning(f"Factory reset completed by {current_user.username}")
|
||||
logger.warning(f"Factory reset completed by {admin_username}")
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
|
||||
Reference in New Issue
Block a user