From 5972b91974c121fac08d46aa04c696589384390a Mon Sep 17 00:00:00 2001 From: Mondo Diaz Date: Fri, 16 Jan 2026 20:45:09 +0000 Subject: [PATCH] Run full integration test suite in CI against deployed environment - Replace 3 smoke tests with full pytest integration suite (~350 tests) - Tests run against deployed Kubernetes environment (feature/stage) - Skip @large and @slow tests in CI for reasonable run times - Production deployments use lightweight smoke tests only (no test data) - Add JUnit report artifacts for test results in GitLab --- .gitlab-ci.yml | 62 ++++++++++++++++++++++++++++++++++++++------------ CHANGELOG.md | 2 ++ 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 38a6514..ec49fb0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,11 +29,45 @@ kics: variables: KICS_CONFIG: kics.config -# Post-deployment integration tests template +# Full integration test suite template (for feature/stage deployments) +# Runs the complete pytest integration test suite against the deployed environment .integration_test_template: &integration_test_template stage: deploy # Runs in deploy stage, but after deployment due to 'needs' image: deps.global.bsf.tools/docker/python:3.12-slim - timeout: 10m + timeout: 20m # Full suite takes longer than smoke tests + variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.pip-cache" + cache: + key: pip-integration-$CI_COMMIT_REF_SLUG + paths: + - .pip-cache/ + policy: pull-push + before_script: + - pip install --index-url "$PIP_INDEX_URL" -r backend/requirements.txt + - pip install --index-url "$PIP_INDEX_URL" pytest pytest-asyncio httpx + script: + - cd backend + # Run full integration test suite, excluding large/slow tests + # ORCHARD_TEST_URL tells the tests which server to connect to + - | + python -m pytest tests/integration/ -v \ + --junitxml=integration-report.xml \ + -m "not large and not slow" \ + --tb=short \ + -x # Stop on first failure for faster feedback + artifacts: + when: always + expire_in: 1 week + paths: + - backend/integration-report.xml + reports: + junit: backend/integration-report.xml + +# Lightweight smoke test template (for production - no test data creation) +.smoke_test_template: &smoke_test_template + stage: deploy + image: deps.global.bsf.tools/docker/python:3.12-slim + timeout: 5m before_script: - pip install --index-url "$PIP_INDEX_URL" httpx script: @@ -43,12 +77,12 @@ kics: import os import sys - BASE_URL = os.environ.get("BASE_URL") + BASE_URL = os.environ.get("ORCHARD_TEST_URL") if not BASE_URL: - print("ERROR: BASE_URL not set") + print("ERROR: ORCHARD_TEST_URL not set") sys.exit(1) - print(f"Running integration tests against {BASE_URL}") + print(f"Running smoke tests against {BASE_URL}") client = httpx.Client(base_url=BASE_URL, timeout=30.0) errors = [] @@ -86,26 +120,26 @@ kics: print(f" FAIL: {e}") sys.exit(1) else: - print("SUCCESS: All integration tests passed!") + print("SUCCESS: All smoke tests passed!") sys.exit(0) PYTEST_SCRIPT -# Integration tests for stage deployment +# Integration tests for stage deployment (full suite) integration_test_stage: <<: *integration_test_template needs: [deploy_stage] variables: - BASE_URL: https://orchard-stage.common.global.bsf.tools + ORCHARD_TEST_URL: https://orchard-stage.common.global.bsf.tools rules: - if: '$CI_COMMIT_BRANCH == "main"' when: on_success -# Integration tests for feature deployment +# Integration tests for feature deployment (full suite) integration_test_feature: <<: *integration_test_template needs: [deploy_feature] variables: - BASE_URL: https://orchard-$CI_COMMIT_REF_SLUG.common.global.bsf.tools + ORCHARD_TEST_URL: https://orchard-$CI_COMMIT_REF_SLUG.common.global.bsf.tools rules: - if: '$CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != "main"' when: on_success @@ -128,7 +162,7 @@ python_tests: - pip install --index-url "$PIP_INDEX_URL" pytest pytest-asyncio pytest-cov httpx script: - cd backend - # Only run unit tests - integration tests require Docker Compose services + # Run unit tests (integration tests run post-deployment against live environment) - python -m pytest tests/unit/ -v --cov=app --cov-report=term --cov-report=xml:coverage.xml --cov-report=html:coverage_html --junitxml=pytest-report.xml artifacts: when: always @@ -357,12 +391,12 @@ deploy_prod: when: manual # Require manual approval for prod allow_failure: false -# Integration tests for production deployment +# Smoke tests for production deployment (read-only, no test data creation) integration_test_prod: - <<: *integration_test_template + <<: *smoke_test_template needs: [deploy_prod] variables: - BASE_URL: https://orchard.common.global.bsf.tools + ORCHARD_TEST_URL: https://orchard.common.global.bsf.tools rules: - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/' when: on_success diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c3f610..11b8c13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added internal proxy configuration for npm, pip, helm, and apt (#51) ### Changed +- CI integration tests now run full pytest suite (~350 tests) against deployed environment instead of 3 smoke tests +- CI production deployment uses lightweight smoke tests only (no test data creation in prod) - Updated download ref resolution to check versions before tags (version → tag → artifact ID) (#56) - Deploy jobs now require all security scans to pass before deployment (added test_image, app_deps_scan, cve_scan, cve_sbom_analysis, app_sbom_analysis to dependencies) (#63) - Increased deploy job timeout from 5m to 10m (#63)