diff --git a/CHANGELOG.md b/CHANGELOG.md index dc14d18..b8c5e2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Added HTTP connection pooling infrastructure for improved PyPI proxy performance + - `HttpClientManager` with configurable pool size, timeouts, and thread pool executor + - Eliminates per-request connection overhead (~100-500ms → ~5ms) +- Added Redis caching layer with category-aware TTL for hermetic builds + - `CacheService` with graceful fallback when Redis unavailable + - Immutable data (artifact metadata, dependencies) cached forever + - Mutable data (package index, versions) uses configurable TTL +- Added `ArtifactRepository` for batch database operations + - `batch_upsert_dependencies()` reduces N+1 queries to single INSERT + - `get_or_create_artifact()` uses atomic ON CONFLICT upsert +- Added infrastructure status to health endpoint (`/health`) + - Reports HTTP pool size and worker threads + - Reports Redis cache connection status +- Added new configuration settings for HTTP client, Redis, and cache TTL + - `ORCHARD_HTTP_MAX_CONNECTIONS`, `ORCHARD_HTTP_CONNECT_TIMEOUT`, etc. + - `ORCHARD_REDIS_HOST`, `ORCHARD_REDIS_PORT`, `ORCHARD_REDIS_ENABLED` + - `ORCHARD_CACHE_TTL_INDEX`, `ORCHARD_CACHE_TTL_VERSIONS`, etc. - Added transparent PyPI proxy implementing PEP 503 Simple API (#108) - `GET /pypi/simple/` - package index (proxied from upstream) - `GET /pypi/simple/{package}/` - version list with rewritten download links @@ -16,6 +33,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `POST /api/v1/cache/resolve` endpoint to cache packages by coordinates instead of URL (#108) ### Changed +- PyPI proxy now uses shared HTTP connection pool instead of per-request clients +- PyPI proxy now caches upstream source configuration in Redis +- Dependency storage now uses batch INSERT instead of individual queries +- Increased default database pool size from 5 to 20 connections +- Increased default database max overflow from 10 to 30 connections +- Enabled Redis in Helm chart values for dev, stage, and prod environments - Upstream sources table text is now centered under column headers (#108) - ENV badge now appears inline with source name instead of separate column (#108) - Test and Edit buttons now have more prominent button styling (#108) diff --git a/backend/tests/integration/test_pypi_proxy.py b/backend/tests/integration/test_pypi_proxy.py index 1354e42..4d8fc84 100644 --- a/backend/tests/integration/test_pypi_proxy.py +++ b/backend/tests/integration/test_pypi_proxy.py @@ -147,6 +147,7 @@ class TestPyPIProxyInfrastructure: assert response.status_code == 200 data = response.json() - assert data["status"] == "healthy" - # Infrastructure status may include these if implemented - # assert "infrastructure" in data + assert data["status"] == "ok" + # Infrastructure status should be present + assert "http_pool" in data + assert "cache" in data diff --git a/backend/tests/unit/test_db_utils.py b/backend/tests/unit/test_db_utils.py index 045882b..b1cd418 100644 --- a/backend/tests/unit/test_db_utils.py +++ b/backend/tests/unit/test_db_utils.py @@ -8,7 +8,7 @@ class TestArtifactRepository: def test_batch_dependency_values_formatting(self): """batch_upsert_dependencies should format values correctly.""" - from backend.app.db_utils import ArtifactRepository + from app.db_utils import ArtifactRepository deps = [ ("_pypi", "numpy", ">=1.21.0"), @@ -29,7 +29,7 @@ class TestArtifactRepository: def test_empty_dependencies_returns_empty_list(self): """Empty dependency list should return empty values.""" - from backend.app.db_utils import ArtifactRepository + from app.db_utils import ArtifactRepository values = ArtifactRepository._format_dependency_values("abc123", []) @@ -37,7 +37,7 @@ class TestArtifactRepository: def test_format_dependency_values_preserves_special_characters(self): """Version constraints with special characters should be preserved.""" - from backend.app.db_utils import ArtifactRepository + from app.db_utils import ArtifactRepository deps = [ ("_pypi", "package-name", ">=1.0.0,<2.0.0"), @@ -51,7 +51,7 @@ class TestArtifactRepository: def test_batch_upsert_dependencies_returns_zero_for_empty(self): """batch_upsert_dependencies should return 0 for empty list without DB call.""" - from backend.app.db_utils import ArtifactRepository + from app.db_utils import ArtifactRepository mock_db = MagicMock() repo = ArtifactRepository(mock_db) @@ -64,8 +64,8 @@ class TestArtifactRepository: def test_get_or_create_artifact_builds_correct_statement(self): """get_or_create_artifact should use ON CONFLICT DO UPDATE.""" - from backend.app.db_utils import ArtifactRepository - from backend.app.models import Artifact + from app.db_utils import ArtifactRepository + from app.models import Artifact mock_db = MagicMock() mock_result = MagicMock() @@ -88,7 +88,7 @@ class TestArtifactRepository: def test_get_or_create_artifact_existing_not_created(self): """get_or_create_artifact should return created=False for existing artifact.""" - from backend.app.db_utils import ArtifactRepository + from app.db_utils import ArtifactRepository mock_db = MagicMock() mock_result = MagicMock() @@ -108,7 +108,7 @@ class TestArtifactRepository: def test_get_cached_url_with_artifact_returns_tuple(self): """get_cached_url_with_artifact should return (CachedUrl, Artifact) tuple.""" - from backend.app.db_utils import ArtifactRepository + from app.db_utils import ArtifactRepository mock_db = MagicMock() mock_cached_url = MagicMock() @@ -125,7 +125,7 @@ class TestArtifactRepository: def test_get_cached_url_with_artifact_returns_none_when_not_found(self): """get_cached_url_with_artifact should return None when URL not cached.""" - from backend.app.db_utils import ArtifactRepository + from app.db_utils import ArtifactRepository mock_db = MagicMock() mock_db.query.return_value.join.return_value.filter.return_value.first.return_value = None @@ -137,7 +137,7 @@ class TestArtifactRepository: def test_get_artifact_dependencies_returns_list(self): """get_artifact_dependencies should return list of dependencies.""" - from backend.app.db_utils import ArtifactRepository + from app.db_utils import ArtifactRepository mock_db = MagicMock() mock_dep1 = MagicMock() @@ -156,7 +156,7 @@ class TestArtifactRepository: def test_get_artifact_dependencies_returns_empty_list(self): """get_artifact_dependencies should return empty list when no dependencies.""" - from backend.app.db_utils import ArtifactRepository + from app.db_utils import ArtifactRepository mock_db = MagicMock() mock_db.query.return_value.filter.return_value.all.return_value = []