Add upstream caching infrastructure and refactor CI pipeline
This commit is contained in:
Mondo Diaz
@@ -1196,3 +1196,246 @@ class TeamMemberResponse(BaseModel):
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Upstream Caching Schemas
|
||||
# =============================================================================
|
||||
|
||||
# Valid source types
|
||||
SOURCE_TYPES = ["npm", "pypi", "maven", "docker", "helm", "nuget", "deb", "rpm", "generic"]
|
||||
|
||||
# Valid auth types
|
||||
AUTH_TYPES = ["none", "basic", "bearer", "api_key"]
|
||||
|
||||
|
||||
class UpstreamSourceCreate(BaseModel):
|
||||
"""Create a new upstream source"""
|
||||
name: str
|
||||
source_type: str = "generic"
|
||||
url: str
|
||||
enabled: bool = False
|
||||
is_public: bool = True
|
||||
auth_type: str = "none"
|
||||
username: Optional[str] = None
|
||||
password: Optional[str] = None # Write-only
|
||||
headers: Optional[dict] = None # Write-only, custom headers
|
||||
priority: int = 100
|
||||
|
||||
@field_validator('name')
|
||||
@classmethod
|
||||
def validate_name(cls, v: str) -> str:
|
||||
v = v.strip()
|
||||
if not v:
|
||||
raise ValueError("name cannot be empty")
|
||||
if len(v) > 255:
|
||||
raise ValueError("name must be 255 characters or less")
|
||||
return v
|
||||
|
||||
@field_validator('source_type')
|
||||
@classmethod
|
||||
def validate_source_type(cls, v: str) -> str:
|
||||
if v not in SOURCE_TYPES:
|
||||
raise ValueError(f"source_type must be one of: {', '.join(SOURCE_TYPES)}")
|
||||
return v
|
||||
|
||||
@field_validator('url')
|
||||
@classmethod
|
||||
def validate_url(cls, v: str) -> str:
|
||||
v = v.strip()
|
||||
if not v:
|
||||
raise ValueError("url cannot be empty")
|
||||
if not (v.startswith('http://') or v.startswith('https://')):
|
||||
raise ValueError("url must start with http:// or https://")
|
||||
if len(v) > 2048:
|
||||
raise ValueError("url must be 2048 characters or less")
|
||||
return v
|
||||
|
||||
@field_validator('auth_type')
|
||||
@classmethod
|
||||
def validate_auth_type(cls, v: str) -> str:
|
||||
if v not in AUTH_TYPES:
|
||||
raise ValueError(f"auth_type must be one of: {', '.join(AUTH_TYPES)}")
|
||||
return v
|
||||
|
||||
@field_validator('priority')
|
||||
@classmethod
|
||||
def validate_priority(cls, v: int) -> int:
|
||||
if v <= 0:
|
||||
raise ValueError("priority must be greater than 0")
|
||||
return v
|
||||
|
||||
|
||||
class UpstreamSourceUpdate(BaseModel):
|
||||
"""Update an upstream source (partial)"""
|
||||
name: Optional[str] = None
|
||||
source_type: Optional[str] = None
|
||||
url: Optional[str] = None
|
||||
enabled: Optional[bool] = None
|
||||
is_public: Optional[bool] = None
|
||||
auth_type: Optional[str] = None
|
||||
username: Optional[str] = None
|
||||
password: Optional[str] = None # Write-only, None = keep existing, empty string = clear
|
||||
headers: Optional[dict] = None # Write-only
|
||||
priority: Optional[int] = None
|
||||
|
||||
@field_validator('name')
|
||||
@classmethod
|
||||
def validate_name(cls, v: Optional[str]) -> Optional[str]:
|
||||
if v is not None:
|
||||
v = v.strip()
|
||||
if not v:
|
||||
raise ValueError("name cannot be empty")
|
||||
if len(v) > 255:
|
||||
raise ValueError("name must be 255 characters or less")
|
||||
return v
|
||||
|
||||
@field_validator('source_type')
|
||||
@classmethod
|
||||
def validate_source_type(cls, v: Optional[str]) -> Optional[str]:
|
||||
if v is not None and v not in SOURCE_TYPES:
|
||||
raise ValueError(f"source_type must be one of: {', '.join(SOURCE_TYPES)}")
|
||||
return v
|
||||
|
||||
@field_validator('url')
|
||||
@classmethod
|
||||
def validate_url(cls, v: Optional[str]) -> Optional[str]:
|
||||
if v is not None:
|
||||
v = v.strip()
|
||||
if not v:
|
||||
raise ValueError("url cannot be empty")
|
||||
if not (v.startswith('http://') or v.startswith('https://')):
|
||||
raise ValueError("url must start with http:// or https://")
|
||||
if len(v) > 2048:
|
||||
raise ValueError("url must be 2048 characters or less")
|
||||
return v
|
||||
|
||||
@field_validator('auth_type')
|
||||
@classmethod
|
||||
def validate_auth_type(cls, v: Optional[str]) -> Optional[str]:
|
||||
if v is not None and v not in AUTH_TYPES:
|
||||
raise ValueError(f"auth_type must be one of: {', '.join(AUTH_TYPES)}")
|
||||
return v
|
||||
|
||||
@field_validator('priority')
|
||||
@classmethod
|
||||
def validate_priority(cls, v: Optional[int]) -> Optional[int]:
|
||||
if v is not None and v <= 0:
|
||||
raise ValueError("priority must be greater than 0")
|
||||
return v
|
||||
|
||||
|
||||
class UpstreamSourceResponse(BaseModel):
|
||||
"""Upstream source response (credentials never included)"""
|
||||
id: UUID
|
||||
name: str
|
||||
source_type: str
|
||||
url: str
|
||||
enabled: bool
|
||||
is_public: bool
|
||||
auth_type: str
|
||||
username: Optional[str]
|
||||
has_password: bool # True if password is set
|
||||
has_headers: bool # True if custom headers are set
|
||||
priority: int
|
||||
source: str = "database" # "database" or "env" (env = defined via environment variables)
|
||||
created_at: Optional[datetime] = None # May be None for legacy/env data
|
||||
updated_at: Optional[datetime] = None # May be None for legacy/env data
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class CacheSettingsResponse(BaseModel):
|
||||
"""Global cache settings response"""
|
||||
allow_public_internet: bool
|
||||
auto_create_system_projects: bool
|
||||
allow_public_internet_env_override: Optional[bool] = None # Set if overridden by env var
|
||||
auto_create_system_projects_env_override: Optional[bool] = None # Set if overridden by env var
|
||||
created_at: Optional[datetime] = None # May be None for legacy data
|
||||
updated_at: Optional[datetime] = None # May be None for legacy data
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class CacheSettingsUpdate(BaseModel):
|
||||
"""Update cache settings (partial)"""
|
||||
allow_public_internet: Optional[bool] = None
|
||||
auto_create_system_projects: Optional[bool] = None
|
||||
|
||||
|
||||
class CachedUrlResponse(BaseModel):
|
||||
"""Cached URL response"""
|
||||
id: UUID
|
||||
url: str
|
||||
url_hash: str
|
||||
artifact_id: str
|
||||
source_id: Optional[UUID]
|
||||
source_name: Optional[str] = None # Populated from join
|
||||
fetched_at: datetime
|
||||
created_at: datetime
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class CacheRequest(BaseModel):
|
||||
"""Request to cache an artifact from an upstream URL"""
|
||||
url: str
|
||||
source_type: str
|
||||
package_name: Optional[str] = None # Auto-derived from URL if not provided
|
||||
tag: Optional[str] = None # Auto-derived from URL if not provided
|
||||
user_project: Optional[str] = None # Cross-reference to user project
|
||||
user_package: Optional[str] = None
|
||||
user_tag: Optional[str] = None
|
||||
expected_hash: Optional[str] = None # Verify downloaded content
|
||||
|
||||
@field_validator('url')
|
||||
@classmethod
|
||||
def validate_url(cls, v: str) -> str:
|
||||
v = v.strip()
|
||||
if not v:
|
||||
raise ValueError("url cannot be empty")
|
||||
if not (v.startswith('http://') or v.startswith('https://')):
|
||||
raise ValueError("url must start with http:// or https://")
|
||||
if len(v) > 4096:
|
||||
raise ValueError("url must be 4096 characters or less")
|
||||
return v
|
||||
|
||||
@field_validator('source_type')
|
||||
@classmethod
|
||||
def validate_source_type(cls, v: str) -> str:
|
||||
if v not in SOURCE_TYPES:
|
||||
raise ValueError(f"source_type must be one of: {', '.join(SOURCE_TYPES)}")
|
||||
return v
|
||||
|
||||
@field_validator('expected_hash')
|
||||
@classmethod
|
||||
def validate_expected_hash(cls, v: Optional[str]) -> Optional[str]:
|
||||
if v is not None:
|
||||
v = v.strip().lower()
|
||||
# Remove sha256: prefix if present
|
||||
if v.startswith('sha256:'):
|
||||
v = v[7:]
|
||||
# Validate hex format
|
||||
if len(v) != 64 or not all(c in '0123456789abcdef' for c in v):
|
||||
raise ValueError("expected_hash must be a 64-character hex string (SHA256)")
|
||||
return v
|
||||
|
||||
|
||||
class CacheResponse(BaseModel):
|
||||
"""Response from caching an artifact"""
|
||||
artifact_id: str
|
||||
sha256: str
|
||||
size: int
|
||||
content_type: Optional[str]
|
||||
already_cached: bool
|
||||
source_url: str
|
||||
source_name: Optional[str]
|
||||
system_project: str
|
||||
system_package: str
|
||||
system_tag: Optional[str]
|
||||
user_reference: Optional[str] = None # e.g., "my-app/npm-deps:lodash-4.17.21"
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user