From 9e3eea4d08e4390635eddb8d9fbb116970a20326 Mon Sep 17 00:00:00 2001 From: Mondo Diaz Date: Thu, 29 Jan 2026 15:33:19 -0600 Subject: [PATCH] Add cache/resolve endpoint and reduce footer padding - Add POST /api/v1/cache/resolve endpoint that caches artifacts by package coordinates (source_type, package, version) instead of requiring a URL - Server finds download URL from configured upstream sources (PyPI supported) - Reduces footer padding from 24px to 12px for cleaner layout --- backend/app/routes.py | 194 +++++++++++++++++++++++++++++ backend/app/schemas.py | 37 ++++++ frontend/src/components/Layout.css | 2 +- 3 files changed, 232 insertions(+), 1 deletion(-) diff --git a/backend/app/routes.py b/backend/app/routes.py index 7c76ee8..e539ef0 100644 --- a/backend/app/routes.py +++ b/backend/app/routes.py @@ -8305,6 +8305,200 @@ def _create_user_cache_reference( return f"{user_project_name}/{user_package_name}" +# --- Cache Resolve Endpoint --- + +from .schemas import CacheResolveRequest + + +@router.post( + "/api/v1/cache/resolve", + response_model=CacheResponse, + tags=["cache"], + summary="Cache an artifact by package coordinates", +) +def cache_resolve( + request: Request, + resolve_request: CacheResolveRequest, + db: Session = Depends(get_db), + storage: S3Storage = Depends(get_storage), + current_user: User = Depends(get_current_user), +): + """ + Cache an artifact by package coordinates (no URL required). + + The server finds the appropriate download URL based on source_type + and configured upstream sources. Currently supports PyPI packages. + + **Request Body:** + - `source_type` (required): Type of source (pypi, npm, maven, etc.) + - `package` (required): Package name + - `version` (required): Package version + - `user_project` (optional): Also create reference in this user project + - `user_package` (optional): Package name in user project + - `user_tag` (optional): Tag name in user project + + **Example (curl):** + ```bash + curl -X POST "http://localhost:8080/api/v1/cache/resolve" \\ + -H "Authorization: Bearer " \\ + -H "Content-Type: application/json" \\ + -d '{ + "source_type": "pypi", + "package": "requests", + "version": "2.31.0" + }' + ``` + """ + import re + import httpx + from urllib.parse import quote, unquote + + if resolve_request.source_type != "pypi": + raise HTTPException( + status_code=501, + detail=f"Cache resolve for '{resolve_request.source_type}' not yet implemented. Currently only 'pypi' is supported." + ) + + # Get PyPI upstream sources + sources = ( + db.query(UpstreamSource) + .filter( + UpstreamSource.source_type == "pypi", + UpstreamSource.enabled == True, + ) + .order_by(UpstreamSource.priority) + .all() + ) + + # Also get env sources + env_sources = [ + s for s in get_env_upstream_sources() + if s.source_type == "pypi" and s.enabled + ] + all_sources = list(sources) + list(env_sources) + all_sources = sorted(all_sources, key=lambda s: s.priority) + + if not all_sources: + raise HTTPException( + status_code=503, + detail="No PyPI upstream sources configured" + ) + + # Normalize package name (PEP 503) + normalized_package = re.sub(r'[-_.]+', '-', resolve_request.package).lower() + + # Query the Simple API to find the download URL + download_url = None + matched_filename = None + last_error = None + + for source in all_sources: + try: + headers = {"User-Agent": "Orchard-CacheResolver/1.0"} + + # Build auth if needed + if hasattr(source, 'auth_type'): + if source.auth_type == "bearer": + password = source.get_password() if hasattr(source, 'get_password') else getattr(source, 'password', None) + if password: + headers["Authorization"] = f"Bearer {password}" + elif source.auth_type == "api_key": + custom_headers = source.get_headers() if hasattr(source, 'get_headers') else {} + if custom_headers: + headers.update(custom_headers) + + auth = None + if hasattr(source, 'auth_type') and source.auth_type == "basic": + username = getattr(source, 'username', None) + if username: + password = source.get_password() if hasattr(source, 'get_password') else getattr(source, 'password', '') + auth = (username, password or '') + + source_url = getattr(source, 'url', '') + package_url = source_url.rstrip('/') + f'/simple/{normalized_package}/' + + timeout = httpx.Timeout(connect=30.0, read=60.0) + + with httpx.Client(timeout=timeout, follow_redirects=True) as client: + response = client.get(package_url, headers=headers, auth=auth) + + if response.status_code == 404: + last_error = f"Package not found in {getattr(source, 'name', 'source')}" + continue + + if response.status_code != 200: + last_error = f"HTTP {response.status_code} from {getattr(source, 'name', 'source')}" + continue + + # Parse HTML to find the version + html = response.text + # Look for links containing the version + # Pattern: href="...{package}-{version}...#sha256=..." + version_pattern = re.escape(resolve_request.version) + link_pattern = rf'href="([^"]+{normalized_package}[^"]*{version_pattern}[^"]*)"' + + matches = re.findall(link_pattern, html, re.IGNORECASE) + + if not matches: + # Try with original package name + link_pattern = rf'href="([^"]+{re.escape(resolve_request.package)}[^"]*{version_pattern}[^"]*)"' + matches = re.findall(link_pattern, html, re.IGNORECASE) + + if matches: + # Prefer .tar.gz or .whl files + for match in matches: + url = match.split('#')[0] # Remove hash fragment + if url.endswith('.tar.gz') or url.endswith('.whl'): + download_url = url + # Extract filename + matched_filename = url.split('/')[-1] + break + if not download_url: + # Use first match + download_url = matches[0].split('#')[0] + matched_filename = download_url.split('/')[-1] + break + + last_error = f"Version {resolve_request.version} not found for {resolve_request.package}" + + except httpx.ConnectError as e: + last_error = f"Connection failed: {e}" + logger.warning(f"Cache resolve: failed to connect to {getattr(source, 'url', 'source')}: {e}") + except httpx.TimeoutException as e: + last_error = f"Timeout: {e}" + logger.warning(f"Cache resolve: timeout connecting to {getattr(source, 'url', 'source')}: {e}") + except Exception as e: + last_error = str(e) + logger.warning(f"Cache resolve: error: {e}") + + if not download_url: + raise HTTPException( + status_code=404, + detail=f"Could not find {resolve_request.package}=={resolve_request.version}: {last_error}" + ) + + # Now cache the artifact using the existing cache_artifact logic + # Construct a CacheRequest + cache_request = CacheRequest( + url=download_url, + source_type="pypi", + package_name=normalized_package, + tag=matched_filename or resolve_request.version, + user_project=resolve_request.user_project, + user_package=resolve_request.user_package, + user_tag=resolve_request.user_tag, + ) + + # Call the cache logic + return cache_artifact( + request=request, + cache_request=cache_request, + db=db, + storage=storage, + current_user=current_user, + ) + + # --- Upstream Sources Admin API --- from .schemas import ( diff --git a/backend/app/schemas.py b/backend/app/schemas.py index b33c019..085c75c 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -1432,4 +1432,41 @@ class CacheResponse(BaseModel): user_reference: Optional[str] = None # e.g., "my-app/npm-deps:lodash-4.17.21" +class CacheResolveRequest(BaseModel): + """Request to cache an artifact by package coordinates (no URL required). + + The server will construct the appropriate URL based on source_type and + configured upstream sources. + """ + source_type: str + package: str + version: str + user_project: Optional[str] = None + user_package: Optional[str] = None + user_tag: Optional[str] = None + + @field_validator('source_type') + @classmethod + def validate_source_type(cls, v: str) -> str: + if v not in SOURCE_TYPES: + raise ValueError(f"source_type must be one of: {', '.join(SOURCE_TYPES)}") + return v + + @field_validator('package') + @classmethod + def validate_package(cls, v: str) -> str: + v = v.strip() + if not v: + raise ValueError("package cannot be empty") + return v + + @field_validator('version') + @classmethod + def validate_version(cls, v: str) -> str: + v = v.strip() + if not v: + raise ValueError("version cannot be empty") + return v + + diff --git a/frontend/src/components/Layout.css b/frontend/src/components/Layout.css index 584719f..d17679d 100644 --- a/frontend/src/components/Layout.css +++ b/frontend/src/components/Layout.css @@ -272,7 +272,7 @@ .footer { background: var(--bg-secondary); border-top: 1px solid var(--border-primary); - padding: 24px 0; + padding: 12px 0; } .footer-content {