feat: add auto-fetch for missing dependencies from upstream registries

Add auto_fetch parameter to dependency resolution endpoint that fetches
missing dependencies from upstream registries (PyPI) when resolving.

- Add RegistryClient abstraction with PyPIRegistryClient implementation
- Extract fetch_and_cache_pypi_package() for reuse
- Add resolve_dependencies_with_fetch() async function
- Extend MissingDependency schema with fetch_attempted/fetch_error
- Add fetched list to DependencyResolutionResponse
- Add auto_fetch_max_depth config setting (default: 3)
- Remove Usage section from Package page UI
- Add 6 integration tests for auto-fetch functionality
This commit is contained in:
Mondo Diaz
2026-02-04 12:01:49 -06:00
parent 9f233e0d4d
commit cbc2e5e11a
10 changed files with 1348 additions and 65 deletions

View File

@@ -141,6 +141,7 @@ from .dependencies import (
get_reverse_dependencies,
check_circular_dependencies,
resolve_dependencies,
resolve_dependencies_with_fetch,
InvalidEnsureFileError,
CircularDependencyError,
DependencyConflictError,
@@ -7025,12 +7026,17 @@ def get_package_reverse_dependencies(
response_model=DependencyResolutionResponse,
tags=["dependencies"],
)
def resolve_artifact_dependencies(
async def resolve_artifact_dependencies(
project_name: str,
package_name: str,
ref: str,
request: Request,
auto_fetch: bool = Query(
False,
description="Fetch missing dependencies from upstream registries (e.g., PyPI)"
),
db: Session = Depends(get_db),
storage: S3Storage = Depends(get_storage),
current_user: Optional[User] = Depends(get_current_user_optional),
):
"""
@@ -7039,6 +7045,16 @@ def resolve_artifact_dependencies(
Returns a flat list of all artifacts needed, in topological order
(dependencies before dependents). Includes download URLs for each artifact.
**Parameters:**
- **auto_fetch**: When true, attempts to fetch missing dependencies from
upstream registries (PyPI for _pypi project packages). Default is false
for fast, network-free resolution.
**Response Fields:**
- **resolved**: All artifacts in dependency order with download URLs
- **missing**: Dependencies that couldn't be resolved (with fetch status if auto_fetch=true)
- **fetched**: Artifacts that were fetched from upstream during this request
**Error Responses:**
- 404: Artifact or dependency not found
- 409: Circular dependency or version conflict detected
@@ -7050,7 +7066,39 @@ def resolve_artifact_dependencies(
base_url = str(request.base_url).rstrip("/")
try:
return resolve_dependencies(db, project_name, package_name, ref, base_url)
if auto_fetch:
# Use async resolution with auto-fetch
from .registry_client import get_registry_client
from .pypi_proxy import _get_pypi_upstream_sources
settings = get_settings()
# Get HTTP client from app state
http_client = request.app.state.http_client.get_client()
# Get upstream sources for registry clients
pypi_sources = _get_pypi_upstream_sources(db)
# Build registry clients
registry_clients = {}
pypi_client = get_registry_client("pypi", http_client, pypi_sources)
if pypi_client:
registry_clients["_pypi"] = pypi_client
return await resolve_dependencies_with_fetch(
db=db,
project_name=project_name,
package_name=package_name,
ref=ref,
base_url=base_url,
storage=storage,
registry_clients=registry_clients,
max_fetch_depth=settings.auto_fetch_max_depth,
)
else:
# Fast, synchronous resolution without network calls
return resolve_dependencies(db, project_name, package_name, ref, base_url)
except DependencyNotFoundError as e:
raise HTTPException(
status_code=404,