perf: cache upstream sources lookup in pypi_proxy
This commit is contained in:
@@ -254,6 +254,62 @@ def _extract_pypi_version(filename: str) -> Optional[str]:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
async def _get_pypi_upstream_sources_cached(
|
||||||
|
db: Session,
|
||||||
|
cache: CacheService,
|
||||||
|
) -> list[UpstreamSource]:
|
||||||
|
"""
|
||||||
|
Get PyPI upstream sources with caching.
|
||||||
|
|
||||||
|
Sources are cached for cache_ttl_upstream seconds to avoid
|
||||||
|
repeated database queries on every request.
|
||||||
|
"""
|
||||||
|
cache_key = "sources"
|
||||||
|
|
||||||
|
# Try cache first
|
||||||
|
cached = await cache.get(CacheCategory.UPSTREAM_SOURCES, cache_key, protocol="pypi")
|
||||||
|
if cached:
|
||||||
|
source_data = json.loads(cached.decode())
|
||||||
|
# Reconstruct UpstreamSource-like objects from cached data
|
||||||
|
# We cache just the essential fields needed for requests
|
||||||
|
return [type('CachedSource', (), d)() for d in source_data]
|
||||||
|
|
||||||
|
# Query database
|
||||||
|
db_sources = (
|
||||||
|
db.query(UpstreamSource)
|
||||||
|
.filter(UpstreamSource.source_type == "pypi", UpstreamSource.enabled == True)
|
||||||
|
.order_by(UpstreamSource.priority)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
# Combine with env sources
|
||||||
|
env_sources = [s for s in get_env_upstream_sources() if s.source_type == "pypi"]
|
||||||
|
all_sources = list(db_sources) + list(env_sources)
|
||||||
|
all_sources = sorted(all_sources, key=lambda s: s.priority)
|
||||||
|
|
||||||
|
# Cache the essential fields
|
||||||
|
if all_sources and cache.enabled:
|
||||||
|
cache_data = [
|
||||||
|
{
|
||||||
|
"name": s.name,
|
||||||
|
"url": s.url,
|
||||||
|
"priority": s.priority,
|
||||||
|
"auth_type": getattr(s, "auth_type", "none"),
|
||||||
|
"username": getattr(s, "username", None),
|
||||||
|
"password": getattr(s, "password", None),
|
||||||
|
}
|
||||||
|
for s in all_sources
|
||||||
|
]
|
||||||
|
await cache.set(
|
||||||
|
CacheCategory.UPSTREAM_SOURCES,
|
||||||
|
cache_key,
|
||||||
|
json.dumps(cache_data).encode(),
|
||||||
|
protocol="pypi",
|
||||||
|
)
|
||||||
|
|
||||||
|
return all_sources
|
||||||
|
|
||||||
|
|
||||||
def _get_pypi_upstream_sources(db: Session) -> list[UpstreamSource]:
|
def _get_pypi_upstream_sources(db: Session) -> list[UpstreamSource]:
|
||||||
"""Get all enabled upstream sources configured for PyPI."""
|
"""Get all enabled upstream sources configured for PyPI."""
|
||||||
# Get database sources
|
# Get database sources
|
||||||
|
|||||||
Reference in New Issue
Block a user