d274f3f375
Replace unbounded thread spawning with managed worker pool:
- New pypi_cache_tasks table tracks caching jobs
- Thread pool with 5 workers (configurable via ORCHARD_PYPI_CACHE_WORKERS)
- Automatic retries with exponential backoff (30s, 60s, then fail)
- Deduplication to prevent duplicate caching attempts
New API endpoints for visibility and control:
- GET /pypi/cache/status - queue health summary
- GET /pypi/cache/failed - list failed tasks with errors
- POST /pypi/cache/retry/{package} - retry single package
- POST /pypi/cache/retry-all - retry all failed packages
This fixes silent failures in background dependency caching where
packages would fail to cache without any tracking or retry mechanism.
873 lines
31 KiB
Python
873 lines
31 KiB
Python
from datetime import datetime
|
|
from sqlalchemy import (
|
|
Column,
|
|
String,
|
|
Text,
|
|
Boolean,
|
|
Integer,
|
|
BigInteger,
|
|
DateTime,
|
|
ForeignKey,
|
|
CheckConstraint,
|
|
Index,
|
|
JSON,
|
|
ARRAY,
|
|
LargeBinary,
|
|
)
|
|
from sqlalchemy.dialects.postgresql import UUID
|
|
from sqlalchemy.orm import relationship, declarative_base
|
|
import uuid
|
|
|
|
Base = declarative_base()
|
|
|
|
|
|
class Project(Base):
|
|
__tablename__ = "projects"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
name = Column(String(255), unique=True, nullable=False)
|
|
description = Column(Text)
|
|
is_public = Column(Boolean, default=True)
|
|
is_system = Column(Boolean, default=False, nullable=False)
|
|
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
updated_at = Column(
|
|
DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow
|
|
)
|
|
created_by = Column(String(255), nullable=False)
|
|
team_id = Column(UUID(as_uuid=True), ForeignKey("teams.id", ondelete="SET NULL"))
|
|
|
|
packages = relationship(
|
|
"Package", back_populates="project", cascade="all, delete-orphan"
|
|
)
|
|
permissions = relationship(
|
|
"AccessPermission", back_populates="project", cascade="all, delete-orphan"
|
|
)
|
|
team = relationship("Team", back_populates="projects")
|
|
|
|
__table_args__ = (
|
|
Index("idx_projects_name", "name"),
|
|
Index("idx_projects_created_by", "created_by"),
|
|
Index("idx_projects_team_id", "team_id"),
|
|
Index("idx_projects_is_system", "is_system"),
|
|
)
|
|
|
|
|
|
class Package(Base):
|
|
__tablename__ = "packages"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
project_id = Column(
|
|
UUID(as_uuid=True),
|
|
ForeignKey("projects.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
)
|
|
name = Column(String(255), nullable=False)
|
|
description = Column(Text)
|
|
format = Column(String(50), default="generic", nullable=False)
|
|
platform = Column(String(50), default="any", nullable=False)
|
|
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
updated_at = Column(
|
|
DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow
|
|
)
|
|
|
|
project = relationship("Project", back_populates="packages")
|
|
tags = relationship("Tag", back_populates="package", cascade="all, delete-orphan")
|
|
uploads = relationship(
|
|
"Upload", back_populates="package", cascade="all, delete-orphan"
|
|
)
|
|
consumers = relationship(
|
|
"Consumer", back_populates="package", cascade="all, delete-orphan"
|
|
)
|
|
versions = relationship(
|
|
"PackageVersion", back_populates="package", cascade="all, delete-orphan"
|
|
)
|
|
|
|
__table_args__ = (
|
|
Index("idx_packages_project_id", "project_id"),
|
|
Index("idx_packages_name", "name"),
|
|
Index("idx_packages_format", "format"),
|
|
Index("idx_packages_platform", "platform"),
|
|
Index(
|
|
"idx_packages_project_name", "project_id", "name", unique=True
|
|
), # Composite unique index
|
|
CheckConstraint(
|
|
"format IN ('generic', 'npm', 'pypi', 'docker', 'deb', 'rpm', 'maven', 'nuget', 'helm')",
|
|
name="check_package_format",
|
|
),
|
|
CheckConstraint(
|
|
"platform IN ('any', 'linux', 'darwin', 'windows', 'linux-amd64', 'linux-arm64', 'darwin-amd64', 'darwin-arm64', 'windows-amd64')",
|
|
name="check_package_platform",
|
|
),
|
|
{"extend_existing": True},
|
|
)
|
|
|
|
|
|
class Artifact(Base):
|
|
__tablename__ = "artifacts"
|
|
|
|
id = Column(String(64), primary_key=True) # SHA256 hash
|
|
size = Column(BigInteger, nullable=False)
|
|
content_type = Column(String(255))
|
|
original_name = Column(String(1024))
|
|
checksum_md5 = Column(String(32)) # MD5 hash for additional verification
|
|
checksum_sha1 = Column(String(40)) # SHA1 hash for compatibility
|
|
s3_etag = Column(String(64)) # S3 ETag for verification
|
|
artifact_metadata = Column(
|
|
"metadata", JSON, default=dict
|
|
) # Format-specific metadata (column name is 'metadata')
|
|
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
created_by = Column(String(255), nullable=False)
|
|
ref_count = Column(Integer, default=1)
|
|
s3_key = Column(String(1024), nullable=False)
|
|
|
|
tags = relationship("Tag", back_populates="artifact")
|
|
uploads = relationship("Upload", back_populates="artifact")
|
|
versions = relationship("PackageVersion", back_populates="artifact")
|
|
dependencies = relationship(
|
|
"ArtifactDependency", back_populates="artifact", cascade="all, delete-orphan"
|
|
)
|
|
|
|
@property
|
|
def sha256(self) -> str:
|
|
"""Alias for id - the SHA256 hash of the artifact content"""
|
|
return self.id
|
|
|
|
@property
|
|
def format_metadata(self):
|
|
"""Alias for artifact_metadata - backward compatibility"""
|
|
return self.artifact_metadata
|
|
|
|
@format_metadata.setter
|
|
def format_metadata(self, value):
|
|
"""Alias setter for artifact_metadata - backward compatibility"""
|
|
self.artifact_metadata = value
|
|
|
|
__table_args__ = (
|
|
Index("idx_artifacts_created_at", "created_at"),
|
|
Index("idx_artifacts_created_by", "created_by"),
|
|
Index("idx_artifacts_ref_count", "ref_count"), # For cleanup queries
|
|
CheckConstraint("ref_count >= 0", name="check_ref_count_non_negative"),
|
|
CheckConstraint("size > 0", name="check_size_positive"),
|
|
)
|
|
|
|
|
|
class Tag(Base):
|
|
__tablename__ = "tags"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
package_id = Column(
|
|
UUID(as_uuid=True),
|
|
ForeignKey("packages.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
)
|
|
name = Column(String(255), nullable=False)
|
|
artifact_id = Column(String(64), ForeignKey("artifacts.id"), nullable=False)
|
|
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
updated_at = Column(
|
|
DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow
|
|
)
|
|
created_by = Column(String(255), nullable=False)
|
|
|
|
package = relationship("Package", back_populates="tags")
|
|
artifact = relationship("Artifact", back_populates="tags")
|
|
history = relationship(
|
|
"TagHistory", back_populates="tag", cascade="all, delete-orphan"
|
|
)
|
|
|
|
__table_args__ = (
|
|
Index("idx_tags_package_id", "package_id"),
|
|
Index("idx_tags_artifact_id", "artifact_id"),
|
|
Index(
|
|
"idx_tags_package_name", "package_id", "name", unique=True
|
|
), # Composite unique index
|
|
Index(
|
|
"idx_tags_package_created_at", "package_id", "created_at"
|
|
), # For recent tags queries
|
|
)
|
|
|
|
|
|
class TagHistory(Base):
|
|
__tablename__ = "tag_history"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
tag_id = Column(
|
|
UUID(as_uuid=True), ForeignKey("tags.id", ondelete="CASCADE"), nullable=False
|
|
)
|
|
old_artifact_id = Column(String(64), ForeignKey("artifacts.id"))
|
|
new_artifact_id = Column(String(64), ForeignKey("artifacts.id"), nullable=False)
|
|
change_type = Column(String(20), nullable=False, default="update")
|
|
changed_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
changed_by = Column(String(255), nullable=False)
|
|
|
|
tag = relationship("Tag", back_populates="history")
|
|
|
|
__table_args__ = (
|
|
Index("idx_tag_history_tag_id", "tag_id"),
|
|
Index("idx_tag_history_changed_at", "changed_at"),
|
|
CheckConstraint(
|
|
"change_type IN ('create', 'update', 'delete')", name="check_change_type"
|
|
),
|
|
)
|
|
|
|
|
|
class PackageVersion(Base):
|
|
"""Immutable version record for a package-artifact relationship.
|
|
|
|
Separates versions (immutable, set at upload) from tags (mutable labels).
|
|
Each artifact in a package can have at most one version.
|
|
"""
|
|
|
|
__tablename__ = "package_versions"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
package_id = Column(
|
|
UUID(as_uuid=True),
|
|
ForeignKey("packages.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
)
|
|
artifact_id = Column(String(64), ForeignKey("artifacts.id"), nullable=False)
|
|
version = Column(String(255), nullable=False)
|
|
version_source = Column(String(50)) # 'explicit', 'filename', 'metadata', 'migrated_from_tag'
|
|
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
created_by = Column(String(255), nullable=False)
|
|
|
|
package = relationship("Package", back_populates="versions")
|
|
artifact = relationship("Artifact", back_populates="versions")
|
|
|
|
__table_args__ = (
|
|
Index("idx_package_versions_package_id", "package_id"),
|
|
Index("idx_package_versions_artifact_id", "artifact_id"),
|
|
Index("idx_package_versions_package_version", "package_id", "version", unique=True),
|
|
Index("idx_package_versions_package_artifact", "package_id", "artifact_id", unique=True),
|
|
)
|
|
|
|
|
|
class Upload(Base):
|
|
__tablename__ = "uploads"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
artifact_id = Column(String(64), ForeignKey("artifacts.id"), nullable=False)
|
|
package_id = Column(UUID(as_uuid=True), ForeignKey("packages.id"), nullable=False)
|
|
original_name = Column(String(1024))
|
|
tag_name = Column(String(255)) # Tag assigned during upload
|
|
user_agent = Column(String(512)) # Client identification
|
|
duration_ms = Column(Integer) # Upload timing in milliseconds
|
|
deduplicated = Column(Boolean, default=False) # Whether artifact was deduplicated
|
|
checksum_verified = Column(Boolean, default=True) # Whether checksum was verified
|
|
status = Column(
|
|
String(20), default="completed", nullable=False
|
|
) # pending, completed, failed
|
|
error_message = Column(Text) # Error details for failed uploads
|
|
client_checksum = Column(String(64)) # Client-provided SHA256 for verification
|
|
uploaded_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
uploaded_by = Column(String(255), nullable=False)
|
|
source_ip = Column(String(45))
|
|
|
|
artifact = relationship("Artifact", back_populates="uploads")
|
|
package = relationship("Package", back_populates="uploads")
|
|
|
|
__table_args__ = (
|
|
Index("idx_uploads_artifact_id", "artifact_id"),
|
|
Index("idx_uploads_package_id", "package_id"),
|
|
Index("idx_uploads_uploaded_at", "uploaded_at"),
|
|
Index("idx_uploads_package_uploaded_at", "package_id", "uploaded_at"),
|
|
Index("idx_uploads_uploaded_by_at", "uploaded_by", "uploaded_at"),
|
|
Index("idx_uploads_status", "status"),
|
|
Index("idx_uploads_status_uploaded_at", "status", "uploaded_at"),
|
|
CheckConstraint(
|
|
"status IN ('pending', 'completed', 'failed')", name="check_upload_status"
|
|
),
|
|
)
|
|
|
|
|
|
class UploadLock(Base):
|
|
"""Track in-progress uploads for conflict detection (409 responses)."""
|
|
|
|
__tablename__ = "upload_locks"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
sha256_hash = Column(String(64), nullable=False)
|
|
package_id = Column(
|
|
UUID(as_uuid=True),
|
|
ForeignKey("packages.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
)
|
|
locked_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
locked_by = Column(String(255), nullable=False)
|
|
expires_at = Column(DateTime(timezone=True), nullable=False)
|
|
|
|
__table_args__ = (
|
|
Index("idx_upload_locks_expires_at", "expires_at"),
|
|
Index(
|
|
"idx_upload_locks_hash_package", "sha256_hash", "package_id", unique=True
|
|
),
|
|
)
|
|
|
|
|
|
class Consumer(Base):
|
|
__tablename__ = "consumers"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
package_id = Column(
|
|
UUID(as_uuid=True),
|
|
ForeignKey("packages.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
)
|
|
project_url = Column(String(2048), nullable=False)
|
|
last_access = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
|
|
package = relationship("Package", back_populates="consumers")
|
|
|
|
__table_args__ = (
|
|
Index("idx_consumers_package_id", "package_id"),
|
|
Index("idx_consumers_last_access", "last_access"),
|
|
)
|
|
|
|
|
|
class AccessPermission(Base):
|
|
__tablename__ = "access_permissions"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
project_id = Column(
|
|
UUID(as_uuid=True),
|
|
ForeignKey("projects.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
)
|
|
user_id = Column(String(255), nullable=False)
|
|
level = Column(String(20), nullable=False)
|
|
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
expires_at = Column(DateTime(timezone=True))
|
|
|
|
project = relationship("Project", back_populates="permissions")
|
|
|
|
__table_args__ = (
|
|
CheckConstraint("level IN ('read', 'write', 'admin')", name="check_level"),
|
|
Index("idx_access_permissions_project_id", "project_id"),
|
|
Index("idx_access_permissions_user_id", "user_id"),
|
|
)
|
|
|
|
|
|
class User(Base):
|
|
"""User account for authentication."""
|
|
|
|
__tablename__ = "users"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
username = Column(String(255), unique=True, nullable=False)
|
|
password_hash = Column(String(255)) # NULL if OIDC-only user
|
|
email = Column(String(255))
|
|
is_admin = Column(Boolean, default=False)
|
|
is_active = Column(Boolean, default=True)
|
|
must_change_password = Column(Boolean, default=False)
|
|
oidc_subject = Column(String(255)) # OIDC subject claim
|
|
oidc_issuer = Column(String(512)) # OIDC issuer URL
|
|
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
updated_at = Column(
|
|
DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow
|
|
)
|
|
last_login = Column(DateTime(timezone=True))
|
|
|
|
# Relationships
|
|
api_keys = relationship(
|
|
"APIKey", back_populates="owner", cascade="all, delete-orphan"
|
|
)
|
|
sessions = relationship(
|
|
"Session", back_populates="user", cascade="all, delete-orphan"
|
|
)
|
|
team_memberships = relationship(
|
|
"TeamMembership", back_populates="user", cascade="all, delete-orphan"
|
|
)
|
|
|
|
__table_args__ = (
|
|
Index("idx_users_username", "username"),
|
|
Index("idx_users_email", "email"),
|
|
Index("idx_users_oidc_subject", "oidc_subject"),
|
|
)
|
|
|
|
|
|
class Session(Base):
|
|
"""User session for web login."""
|
|
|
|
__tablename__ = "sessions"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
user_id = Column(
|
|
UUID(as_uuid=True),
|
|
ForeignKey("users.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
)
|
|
token_hash = Column(String(64), unique=True, nullable=False)
|
|
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
expires_at = Column(DateTime(timezone=True), nullable=False)
|
|
last_accessed = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
user_agent = Column(String(512))
|
|
ip_address = Column(String(45))
|
|
|
|
user = relationship("User", back_populates="sessions")
|
|
|
|
__table_args__ = (
|
|
Index("idx_sessions_user_id", "user_id"),
|
|
Index("idx_sessions_token_hash", "token_hash"),
|
|
Index("idx_sessions_expires_at", "expires_at"),
|
|
)
|
|
|
|
|
|
class AuthSettings(Base):
|
|
"""Authentication settings for OIDC configuration."""
|
|
|
|
__tablename__ = "auth_settings"
|
|
|
|
key = Column(String(255), primary_key=True)
|
|
value = Column(Text, nullable=False)
|
|
updated_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
|
|
|
|
class APIKey(Base):
|
|
__tablename__ = "api_keys"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
key_hash = Column(String(64), unique=True, nullable=False)
|
|
name = Column(String(255), nullable=False)
|
|
user_id = Column(
|
|
String(255), nullable=False
|
|
) # Legacy field, kept for compatibility
|
|
owner_id = Column(
|
|
UUID(as_uuid=True),
|
|
ForeignKey("users.id", ondelete="CASCADE"),
|
|
nullable=True, # Nullable for migration compatibility
|
|
)
|
|
description = Column(Text)
|
|
scopes = Column(ARRAY(String), default=["read", "write"])
|
|
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
expires_at = Column(DateTime(timezone=True))
|
|
last_used = Column(DateTime(timezone=True))
|
|
|
|
owner = relationship("User", back_populates="api_keys")
|
|
|
|
__table_args__ = (
|
|
Index("idx_api_keys_user_id", "user_id"),
|
|
Index("idx_api_keys_key_hash", "key_hash"),
|
|
Index("idx_api_keys_owner_id", "owner_id"),
|
|
)
|
|
|
|
|
|
class AuditLog(Base):
|
|
__tablename__ = "audit_logs"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
action = Column(String(100), nullable=False)
|
|
resource = Column(String(1024), nullable=False)
|
|
user_id = Column(String(255), nullable=False)
|
|
details = Column(JSON)
|
|
timestamp = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
source_ip = Column(String(45))
|
|
|
|
__table_args__ = (
|
|
Index("idx_audit_logs_action", "action"),
|
|
Index("idx_audit_logs_resource", "resource"),
|
|
Index("idx_audit_logs_user_id", "user_id"),
|
|
Index("idx_audit_logs_timestamp", "timestamp"),
|
|
Index("idx_audit_logs_resource_timestamp", "resource", "timestamp"),
|
|
Index("idx_audit_logs_user_timestamp", "user_id", "timestamp"),
|
|
)
|
|
|
|
|
|
class ProjectHistory(Base):
|
|
"""Track changes to project metadata over time."""
|
|
|
|
__tablename__ = "project_history"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
project_id = Column(
|
|
UUID(as_uuid=True),
|
|
ForeignKey("projects.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
)
|
|
field_name = Column(String(100), nullable=False)
|
|
old_value = Column(Text)
|
|
new_value = Column(Text)
|
|
changed_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
changed_by = Column(String(255), nullable=False)
|
|
|
|
__table_args__ = (
|
|
Index("idx_project_history_project_id", "project_id"),
|
|
Index("idx_project_history_changed_at", "changed_at"),
|
|
Index("idx_project_history_project_changed_at", "project_id", "changed_at"),
|
|
)
|
|
|
|
|
|
class PackageHistory(Base):
|
|
"""Track changes to package metadata over time."""
|
|
|
|
__tablename__ = "package_history"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
package_id = Column(
|
|
UUID(as_uuid=True),
|
|
ForeignKey("packages.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
)
|
|
field_name = Column(String(100), nullable=False)
|
|
old_value = Column(Text)
|
|
new_value = Column(Text)
|
|
changed_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
changed_by = Column(String(255), nullable=False)
|
|
|
|
__table_args__ = (
|
|
Index("idx_package_history_package_id", "package_id"),
|
|
Index("idx_package_history_changed_at", "changed_at"),
|
|
Index("idx_package_history_package_changed_at", "package_id", "changed_at"),
|
|
)
|
|
|
|
|
|
class ArtifactDependency(Base):
|
|
"""Dependency declared by an artifact on another package.
|
|
|
|
Each artifact can declare dependencies on other packages, specifying either
|
|
an exact version or a tag. This enables recursive dependency resolution.
|
|
"""
|
|
|
|
__tablename__ = "artifact_dependencies"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
artifact_id = Column(
|
|
String(64),
|
|
ForeignKey("artifacts.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
)
|
|
dependency_project = Column(String(255), nullable=False)
|
|
dependency_package = Column(String(255), nullable=False)
|
|
version_constraint = Column(String(255), nullable=True)
|
|
tag_constraint = Column(String(255), nullable=True)
|
|
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
|
|
# Relationship to the artifact that declares this dependency
|
|
artifact = relationship("Artifact", back_populates="dependencies")
|
|
|
|
__table_args__ = (
|
|
# Exactly one of version_constraint or tag_constraint must be set
|
|
CheckConstraint(
|
|
"(version_constraint IS NOT NULL AND tag_constraint IS NULL) OR "
|
|
"(version_constraint IS NULL AND tag_constraint IS NOT NULL)",
|
|
name="check_constraint_type",
|
|
),
|
|
# Each artifact can only depend on a specific project/package once
|
|
Index(
|
|
"idx_artifact_dependencies_artifact_id",
|
|
"artifact_id",
|
|
),
|
|
Index(
|
|
"idx_artifact_dependencies_target",
|
|
"dependency_project",
|
|
"dependency_package",
|
|
),
|
|
Index(
|
|
"idx_artifact_dependencies_unique",
|
|
"artifact_id",
|
|
"dependency_project",
|
|
"dependency_package",
|
|
unique=True,
|
|
),
|
|
)
|
|
|
|
|
|
class Team(Base):
|
|
"""Team for organizing projects and users."""
|
|
|
|
__tablename__ = "teams"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
name = Column(String(255), nullable=False)
|
|
slug = Column(String(255), unique=True, nullable=False)
|
|
description = Column(Text)
|
|
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
updated_at = Column(
|
|
DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow
|
|
)
|
|
created_by = Column(String(255), nullable=False)
|
|
settings = Column(JSON, default=dict)
|
|
|
|
# Relationships
|
|
memberships = relationship(
|
|
"TeamMembership", back_populates="team", cascade="all, delete-orphan"
|
|
)
|
|
projects = relationship("Project", back_populates="team")
|
|
|
|
__table_args__ = (
|
|
Index("idx_teams_slug", "slug"),
|
|
Index("idx_teams_created_by", "created_by"),
|
|
Index("idx_teams_created_at", "created_at"),
|
|
CheckConstraint(
|
|
"slug ~ '^[a-z0-9][a-z0-9-]*[a-z0-9]$' OR slug ~ '^[a-z0-9]$'",
|
|
name="check_team_slug_format",
|
|
),
|
|
)
|
|
|
|
|
|
class TeamMembership(Base):
|
|
"""Maps users to teams with their roles."""
|
|
|
|
__tablename__ = "team_memberships"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
team_id = Column(
|
|
UUID(as_uuid=True),
|
|
ForeignKey("teams.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
)
|
|
user_id = Column(
|
|
UUID(as_uuid=True),
|
|
ForeignKey("users.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
)
|
|
role = Column(String(20), nullable=False, default="member")
|
|
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
invited_by = Column(String(255))
|
|
|
|
# Relationships
|
|
team = relationship("Team", back_populates="memberships")
|
|
user = relationship("User", back_populates="team_memberships")
|
|
|
|
__table_args__ = (
|
|
Index("idx_team_memberships_team_id", "team_id"),
|
|
Index("idx_team_memberships_user_id", "user_id"),
|
|
Index("idx_team_memberships_role", "role"),
|
|
Index("idx_team_memberships_team_role", "team_id", "role"),
|
|
Index("idx_team_memberships_unique", "team_id", "user_id", unique=True),
|
|
CheckConstraint(
|
|
"role IN ('owner', 'admin', 'member')",
|
|
name="check_team_role",
|
|
),
|
|
)
|
|
|
|
|
|
# =============================================================================
|
|
# Upstream Caching Models
|
|
# =============================================================================
|
|
|
|
# Valid source types for upstream registries
|
|
SOURCE_TYPES = ["npm", "pypi", "maven", "docker", "helm", "nuget", "deb", "rpm", "generic"]
|
|
|
|
# Valid authentication types
|
|
AUTH_TYPES = ["none", "basic", "bearer", "api_key"]
|
|
|
|
|
|
class UpstreamSource(Base):
|
|
"""Configuration for an upstream artifact registry.
|
|
|
|
Stores connection details and authentication for upstream registries
|
|
like npm, PyPI, Maven Central, or private Artifactory instances.
|
|
"""
|
|
|
|
__tablename__ = "upstream_sources"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
name = Column(String(255), unique=True, nullable=False)
|
|
source_type = Column(String(50), default="generic", nullable=False)
|
|
url = Column(String(2048), nullable=False)
|
|
enabled = Column(Boolean, default=False, nullable=False)
|
|
auth_type = Column(String(20), default="none", nullable=False)
|
|
username = Column(String(255))
|
|
password_encrypted = Column(LargeBinary)
|
|
headers_encrypted = Column(LargeBinary)
|
|
priority = Column(Integer, default=100, nullable=False)
|
|
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
updated_at = Column(
|
|
DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow
|
|
)
|
|
|
|
# Relationships
|
|
cached_urls = relationship("CachedUrl", back_populates="source")
|
|
|
|
__table_args__ = (
|
|
Index("idx_upstream_sources_enabled", "enabled"),
|
|
Index("idx_upstream_sources_source_type", "source_type"),
|
|
Index("idx_upstream_sources_priority", "priority"),
|
|
CheckConstraint(
|
|
"source_type IN ('npm', 'pypi', 'maven', 'docker', 'helm', 'nuget', 'deb', 'rpm', 'generic')",
|
|
name="check_source_type",
|
|
),
|
|
CheckConstraint(
|
|
"auth_type IN ('none', 'basic', 'bearer', 'api_key')",
|
|
name="check_auth_type",
|
|
),
|
|
CheckConstraint("priority > 0", name="check_priority_positive"),
|
|
)
|
|
|
|
def set_password(self, password: str) -> None:
|
|
"""Encrypt and store a password/token."""
|
|
from .encryption import encrypt_value
|
|
|
|
if password:
|
|
self.password_encrypted = encrypt_value(password)
|
|
else:
|
|
self.password_encrypted = None
|
|
|
|
def get_password(self) -> str | None:
|
|
"""Decrypt and return the stored password/token."""
|
|
from .encryption import decrypt_value
|
|
|
|
if self.password_encrypted:
|
|
try:
|
|
return decrypt_value(self.password_encrypted)
|
|
except Exception:
|
|
return None
|
|
return None
|
|
|
|
def has_password(self) -> bool:
|
|
"""Check if a password/token is stored."""
|
|
return self.password_encrypted is not None
|
|
|
|
def set_headers(self, headers: dict) -> None:
|
|
"""Encrypt and store custom headers as JSON."""
|
|
from .encryption import encrypt_value
|
|
import json
|
|
|
|
if headers:
|
|
self.headers_encrypted = encrypt_value(json.dumps(headers))
|
|
else:
|
|
self.headers_encrypted = None
|
|
|
|
def get_headers(self) -> dict | None:
|
|
"""Decrypt and return custom headers."""
|
|
from .encryption import decrypt_value
|
|
import json
|
|
|
|
if self.headers_encrypted:
|
|
try:
|
|
return json.loads(decrypt_value(self.headers_encrypted))
|
|
except Exception:
|
|
return None
|
|
return None
|
|
|
|
|
|
class CacheSettings(Base):
|
|
"""Global cache settings (singleton table).
|
|
|
|
Controls behavior of the upstream caching system.
|
|
"""
|
|
|
|
__tablename__ = "cache_settings"
|
|
|
|
id = Column(Integer, primary_key=True, default=1)
|
|
auto_create_system_projects = Column(Boolean, default=True, nullable=False)
|
|
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
updated_at = Column(
|
|
DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow
|
|
)
|
|
|
|
__table_args__ = (
|
|
CheckConstraint("id = 1", name="check_cache_settings_singleton"),
|
|
)
|
|
|
|
|
|
class CachedUrl(Base):
|
|
"""Tracks URL to artifact mappings for provenance.
|
|
|
|
Records which URLs have been cached and maps them to their stored artifacts.
|
|
Enables "is this URL already cached?" lookups and audit trails.
|
|
"""
|
|
|
|
__tablename__ = "cached_urls"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
url = Column(String(4096), nullable=False)
|
|
url_hash = Column(String(64), unique=True, nullable=False)
|
|
artifact_id = Column(
|
|
String(64), ForeignKey("artifacts.id"), nullable=False
|
|
)
|
|
source_id = Column(
|
|
UUID(as_uuid=True),
|
|
ForeignKey("upstream_sources.id", ondelete="SET NULL"),
|
|
)
|
|
fetched_at = Column(DateTime(timezone=True), default=datetime.utcnow, nullable=False)
|
|
response_headers = Column(JSON, default=dict)
|
|
created_at = Column(DateTime(timezone=True), default=datetime.utcnow)
|
|
|
|
# Relationships
|
|
artifact = relationship("Artifact")
|
|
source = relationship("UpstreamSource", back_populates="cached_urls")
|
|
|
|
__table_args__ = (
|
|
Index("idx_cached_urls_url_hash", "url_hash"),
|
|
Index("idx_cached_urls_artifact_id", "artifact_id"),
|
|
Index("idx_cached_urls_source_id", "source_id"),
|
|
Index("idx_cached_urls_fetched_at", "fetched_at"),
|
|
)
|
|
|
|
@staticmethod
|
|
def compute_url_hash(url: str) -> str:
|
|
"""Compute SHA256 hash of a URL for fast lookups."""
|
|
import hashlib
|
|
return hashlib.sha256(url.encode("utf-8")).hexdigest()
|
|
|
|
|
|
class PyPICacheTask(Base):
|
|
"""Task for caching a PyPI package and its dependencies.
|
|
|
|
Tracks the status of background caching operations with retry support.
|
|
Used by the PyPI proxy to ensure reliable dependency caching.
|
|
"""
|
|
|
|
__tablename__ = "pypi_cache_tasks"
|
|
|
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
|
|
|
# What to cache
|
|
package_name = Column(String(255), nullable=False)
|
|
version_constraint = Column(String(255))
|
|
|
|
# Origin tracking
|
|
parent_task_id = Column(
|
|
UUID(as_uuid=True),
|
|
ForeignKey("pypi_cache_tasks.id", ondelete="SET NULL"),
|
|
)
|
|
depth = Column(Integer, nullable=False, default=0)
|
|
triggered_by_artifact = Column(
|
|
String(64),
|
|
ForeignKey("artifacts.id", ondelete="SET NULL"),
|
|
)
|
|
|
|
# Status
|
|
status = Column(String(20), nullable=False, default="pending")
|
|
attempts = Column(Integer, nullable=False, default=0)
|
|
max_attempts = Column(Integer, nullable=False, default=3)
|
|
|
|
# Results
|
|
cached_artifact_id = Column(
|
|
String(64),
|
|
ForeignKey("artifacts.id", ondelete="SET NULL"),
|
|
)
|
|
error_message = Column(Text)
|
|
|
|
# Timing
|
|
created_at = Column(DateTime(timezone=True), nullable=False, default=datetime.utcnow)
|
|
started_at = Column(DateTime(timezone=True))
|
|
completed_at = Column(DateTime(timezone=True))
|
|
next_retry_at = Column(DateTime(timezone=True))
|
|
|
|
# Relationships
|
|
parent_task = relationship(
|
|
"PyPICacheTask",
|
|
remote_side=[id],
|
|
backref="child_tasks",
|
|
)
|
|
|
|
__table_args__ = (
|
|
Index("idx_pypi_cache_tasks_status_retry", "status", "next_retry_at"),
|
|
Index("idx_pypi_cache_tasks_package_status", "package_name", "status"),
|
|
Index("idx_pypi_cache_tasks_parent", "parent_task_id"),
|
|
Index("idx_pypi_cache_tasks_triggered_by", "triggered_by_artifact"),
|
|
Index("idx_pypi_cache_tasks_cached_artifact", "cached_artifact_id"),
|
|
Index("idx_pypi_cache_tasks_depth_created", "depth", "created_at"),
|
|
CheckConstraint(
|
|
"status IN ('pending', 'in_progress', 'completed', 'failed')",
|
|
name="check_task_status",
|
|
),
|
|
CheckConstraint("depth >= 0", name="check_depth_non_negative"),
|
|
CheckConstraint("attempts >= 0", name="check_attempts_non_negative"),
|
|
)
|
|
|
|
|