feat: add auto-fetch for missing dependencies from upstream registries
Add auto_fetch parameter to dependency resolution endpoint that fetches missing dependencies from upstream registries (PyPI) when resolving. - Add RegistryClient abstraction with PyPIRegistryClient implementation - Extract fetch_and_cache_pypi_package() for reuse - Add resolve_dependencies_with_fetch() async function - Extend MissingDependency schema with fetch_attempted/fetch_error - Add fetched list to DependencyResolutionResponse - Add auto_fetch_max_depth config setting (default: 3) - Remove Usage section from Package page UI - Add 6 integration tests for auto-fetch functionality
This commit is contained in:
Mondo Diaz
@@ -1067,3 +1067,277 @@ class TestConflictDetection:
|
||||
finally:
|
||||
for pkg in [pkg_app, pkg_lib_a, pkg_lib_b, pkg_common]:
|
||||
integration_client.delete(f"/api/v1/project/{test_project}/packages/{pkg}")
|
||||
|
||||
|
||||
class TestAutoFetchDependencies:
|
||||
"""Tests for auto-fetch functionality in dependency resolution.
|
||||
|
||||
These tests verify:
|
||||
- Resolution with auto_fetch=false (default) behavior is unchanged
|
||||
- Resolution with auto_fetch=true attempts to fetch missing dependencies
|
||||
- Proper handling of missing/non-existent packages
|
||||
- Response schema includes fetched artifacts list
|
||||
"""
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_resolve_auto_fetch_false_is_default(
|
||||
self, integration_client, test_package, unique_test_id
|
||||
):
|
||||
"""Test that auto_fetch=false is the default and behaves as before."""
|
||||
project_name, package_name = test_package
|
||||
|
||||
# Upload a simple artifact without dependencies
|
||||
content = unique_content("autofetch-default", unique_test_id, "nodeps")
|
||||
files = {"file": ("default.tar.gz", BytesIO(content), "application/gzip")}
|
||||
response = integration_client.post(
|
||||
f"/api/v1/project/{project_name}/{package_name}/upload",
|
||||
files=files,
|
||||
data={"version": f"v1.0.0-{unique_test_id}"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# Resolve without auto_fetch param (should default to false)
|
||||
response = integration_client.get(
|
||||
f"/api/v1/project/{project_name}/{package_name}/+/v1.0.0-{unique_test_id}/resolve"
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Should have empty fetched list
|
||||
assert data.get("fetched", []) == []
|
||||
assert data["artifact_count"] == 1
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_resolve_auto_fetch_explicit_false(
|
||||
self, integration_client, test_package, unique_test_id
|
||||
):
|
||||
"""Test that auto_fetch=false works explicitly."""
|
||||
project_name, package_name = test_package
|
||||
|
||||
content = unique_content("autofetch-explicit-false", unique_test_id, "nodeps")
|
||||
files = {"file": ("explicit.tar.gz", BytesIO(content), "application/gzip")}
|
||||
response = integration_client.post(
|
||||
f"/api/v1/project/{project_name}/{package_name}/upload",
|
||||
files=files,
|
||||
data={"version": f"v2.0.0-{unique_test_id}"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# Resolve with explicit auto_fetch=false
|
||||
response = integration_client.get(
|
||||
f"/api/v1/project/{project_name}/{package_name}/+/v2.0.0-{unique_test_id}/resolve",
|
||||
params={"auto_fetch": "false"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data.get("fetched", []) == []
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_resolve_auto_fetch_true_no_missing_deps(
|
||||
self, integration_client, test_project, unique_test_id
|
||||
):
|
||||
"""Test that auto_fetch=true works when all deps are already cached."""
|
||||
pkg_a = f"fetch-a-{unique_test_id}"
|
||||
pkg_b = f"fetch-b-{unique_test_id}"
|
||||
|
||||
for pkg in [pkg_a, pkg_b]:
|
||||
response = integration_client.post(
|
||||
f"/api/v1/project/{test_project}/packages",
|
||||
json={"name": pkg}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
try:
|
||||
# Upload B (no deps)
|
||||
content_b = unique_content("B", unique_test_id, "fetch")
|
||||
files = {"file": ("b.tar.gz", BytesIO(content_b), "application/gzip")}
|
||||
response = integration_client.post(
|
||||
f"/api/v1/project/{test_project}/{pkg_b}/upload",
|
||||
files=files,
|
||||
data={"version": "1.0.0"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# Upload A (depends on B)
|
||||
ensure_a = yaml.dump({
|
||||
"dependencies": [
|
||||
{"project": test_project, "package": pkg_b, "version": "1.0.0"}
|
||||
]
|
||||
})
|
||||
content_a = unique_content("A", unique_test_id, "fetch")
|
||||
files = {
|
||||
"file": ("a.tar.gz", BytesIO(content_a), "application/gzip"),
|
||||
"ensure": ("orchard.ensure", BytesIO(ensure_a.encode()), "application/x-yaml"),
|
||||
}
|
||||
response = integration_client.post(
|
||||
f"/api/v1/project/{test_project}/{pkg_a}/upload",
|
||||
files=files,
|
||||
data={"version": "1.0.0"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# Resolve with auto_fetch=true - should work since deps are cached
|
||||
response = integration_client.get(
|
||||
f"/api/v1/project/{test_project}/{pkg_a}/+/1.0.0/resolve",
|
||||
params={"auto_fetch": "true"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Should resolve successfully
|
||||
assert data["artifact_count"] == 2
|
||||
# Nothing fetched since everything was cached
|
||||
assert len(data.get("fetched", [])) == 0
|
||||
# No missing deps
|
||||
assert len(data.get("missing", [])) == 0
|
||||
|
||||
finally:
|
||||
for pkg in [pkg_a, pkg_b]:
|
||||
integration_client.delete(f"/api/v1/project/{test_project}/packages/{pkg}")
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_resolve_missing_dep_with_auto_fetch_false(
|
||||
self, integration_client, test_package, unique_test_id
|
||||
):
|
||||
"""Test that missing deps are reported when auto_fetch=false."""
|
||||
project_name, package_name = test_package
|
||||
|
||||
# Create _pypi system project if it doesn't exist
|
||||
response = integration_client.get("/api/v1/projects/_pypi")
|
||||
if response.status_code == 404:
|
||||
response = integration_client.post(
|
||||
"/api/v1/projects",
|
||||
json={"name": "_pypi", "description": "System project for PyPI packages"}
|
||||
)
|
||||
# May fail if already exists or can't create - that's ok
|
||||
|
||||
# Upload artifact with dependency on _pypi package that doesn't exist locally
|
||||
ensure_content = yaml.dump({
|
||||
"dependencies": [
|
||||
{"project": "_pypi", "package": "nonexistent-pkg-xyz123", "version": ">=1.0.0"}
|
||||
]
|
||||
})
|
||||
|
||||
content = unique_content("missing-pypi", unique_test_id, "dep")
|
||||
files = {
|
||||
"file": ("missing-pypi-dep.tar.gz", BytesIO(content), "application/gzip"),
|
||||
"ensure": ("orchard.ensure", BytesIO(ensure_content.encode()), "application/x-yaml"),
|
||||
}
|
||||
response = integration_client.post(
|
||||
f"/api/v1/project/{project_name}/{package_name}/upload",
|
||||
files=files,
|
||||
data={"version": f"v3.0.0-{unique_test_id}"},
|
||||
)
|
||||
# Upload should succeed - validation is loose for system projects
|
||||
if response.status_code == 200:
|
||||
# Resolve without auto_fetch - should report missing
|
||||
response = integration_client.get(
|
||||
f"/api/v1/project/{project_name}/{package_name}/+/v3.0.0-{unique_test_id}/resolve",
|
||||
params={"auto_fetch": "false"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Should have missing dependencies
|
||||
assert len(data.get("missing", [])) >= 1
|
||||
|
||||
# Verify missing dependency structure
|
||||
missing = data["missing"][0]
|
||||
assert missing["project"] == "_pypi"
|
||||
assert missing["package"] == "nonexistent-pkg-xyz123"
|
||||
# Without auto_fetch, these should be false/None
|
||||
assert missing.get("fetch_attempted", False) is False
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_resolve_response_schema_has_fetched_field(
|
||||
self, integration_client, test_package, unique_test_id
|
||||
):
|
||||
"""Test that the resolve response always includes the fetched field."""
|
||||
project_name, package_name = test_package
|
||||
|
||||
content = unique_content("schema-check", unique_test_id, "nodeps")
|
||||
files = {"file": ("schema.tar.gz", BytesIO(content), "application/gzip")}
|
||||
response = integration_client.post(
|
||||
f"/api/v1/project/{project_name}/{package_name}/upload",
|
||||
files=files,
|
||||
data={"version": f"v4.0.0-{unique_test_id}"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# Check both auto_fetch modes include fetched field
|
||||
for auto_fetch in ["false", "true"]:
|
||||
response = integration_client.get(
|
||||
f"/api/v1/project/{project_name}/{package_name}/+/v4.0.0-{unique_test_id}/resolve",
|
||||
params={"auto_fetch": auto_fetch},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Required fields
|
||||
assert "requested" in data
|
||||
assert "resolved" in data
|
||||
assert "missing" in data
|
||||
assert "fetched" in data # New field
|
||||
assert "total_size" in data
|
||||
assert "artifact_count" in data
|
||||
|
||||
# Types
|
||||
assert isinstance(data["fetched"], list)
|
||||
assert isinstance(data["missing"], list)
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_missing_dep_schema_has_fetch_fields(
|
||||
self, integration_client, test_package, unique_test_id
|
||||
):
|
||||
"""Test that missing dependency entries have fetch_attempted and fetch_error fields."""
|
||||
project_name, package_name = test_package
|
||||
|
||||
# Create a dependency on a non-existent package in a real project
|
||||
dep_project_name = f"dep-test-{unique_test_id}"
|
||||
response = integration_client.post(
|
||||
"/api/v1/projects", json={"name": dep_project_name}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
try:
|
||||
ensure_content = yaml.dump({
|
||||
"dependencies": [
|
||||
{"project": dep_project_name, "package": "nonexistent-pkg", "version": "1.0.0"}
|
||||
]
|
||||
})
|
||||
|
||||
content = unique_content("missing-schema", unique_test_id, "check")
|
||||
files = {
|
||||
"file": ("missing-schema.tar.gz", BytesIO(content), "application/gzip"),
|
||||
"ensure": ("orchard.ensure", BytesIO(ensure_content.encode()), "application/x-yaml"),
|
||||
}
|
||||
response = integration_client.post(
|
||||
f"/api/v1/project/{project_name}/{package_name}/upload",
|
||||
files=files,
|
||||
data={"version": f"v5.0.0-{unique_test_id}"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
# Resolve
|
||||
response = integration_client.get(
|
||||
f"/api/v1/project/{project_name}/{package_name}/+/v5.0.0-{unique_test_id}/resolve",
|
||||
params={"auto_fetch": "true"},
|
||||
)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
|
||||
# Should have missing dependencies
|
||||
assert len(data.get("missing", [])) >= 1
|
||||
|
||||
# Check schema for missing dependency
|
||||
missing = data["missing"][0]
|
||||
assert "project" in missing
|
||||
assert "package" in missing
|
||||
assert "constraint" in missing
|
||||
assert "required_by" in missing
|
||||
# New fields
|
||||
assert "fetch_attempted" in missing
|
||||
assert "fetch_error" in missing # May be None
|
||||
|
||||
finally:
|
||||
integration_client.delete(f"/api/v1/projects/{dep_project_name}")
|
||||
|
||||
Reference in New Issue
Block a user