Add robust PyPI dependency caching with task queue
Replace unbounded thread spawning with managed worker pool:
- New pypi_cache_tasks table tracks caching jobs
- Thread pool with 5 workers (configurable via ORCHARD_PYPI_CACHE_WORKERS)
- Automatic retries with exponential backoff (30s, 60s, then fail)
- Deduplication to prevent duplicate caching attempts
New API endpoints for visibility and control:
- GET /pypi/cache/status - queue health summary
- GET /pypi/cache/failed - list failed tasks with errors
- POST /pypi/cache/retry/{package} - retry single package
- POST /pypi/cache/retry-all - retry all failed packages
This fixes silent failures in background dependency caching where
packages would fail to cache without any tracking or retry mechanism.
This commit is contained in:
@@ -64,6 +64,11 @@ class Settings(BaseSettings):
|
||||
# Global cache settings override (None = use DB value, True/False = override DB)
|
||||
cache_auto_create_system_projects: Optional[bool] = None # Override auto_create_system_projects
|
||||
|
||||
# PyPI Cache Worker settings
|
||||
pypi_cache_workers: int = 5 # Number of concurrent cache workers
|
||||
pypi_cache_max_depth: int = 10 # Maximum recursion depth for dependency caching
|
||||
pypi_cache_max_attempts: int = 3 # Maximum retry attempts for failed cache tasks
|
||||
|
||||
# JWT Authentication settings (optional, for external identity providers)
|
||||
jwt_enabled: bool = False # Enable JWT token validation
|
||||
jwt_secret: str = "" # Secret key for HS256, or leave empty for RS256 with JWKS
|
||||
@@ -88,6 +93,24 @@ class Settings(BaseSettings):
|
||||
def is_production(self) -> bool:
|
||||
return self.env.lower() == "production"
|
||||
|
||||
@property
|
||||
def PORT(self) -> int:
|
||||
"""Alias for server_port for compatibility."""
|
||||
return self.server_port
|
||||
|
||||
# Uppercase aliases for PyPI cache settings (for backward compatibility)
|
||||
@property
|
||||
def PYPI_CACHE_WORKERS(self) -> int:
|
||||
return self.pypi_cache_workers
|
||||
|
||||
@property
|
||||
def PYPI_CACHE_MAX_DEPTH(self) -> int:
|
||||
return self.pypi_cache_max_depth
|
||||
|
||||
@property
|
||||
def PYPI_CACHE_MAX_ATTEMPTS(self) -> int:
|
||||
return self.pypi_cache_max_attempts
|
||||
|
||||
class Config:
|
||||
env_prefix = "ORCHARD_"
|
||||
case_sensitive = False
|
||||
|
||||
Reference in New Issue
Block a user