From 2d14d0f2f9ee60dd66a71a021e8e3aeda90c374c Mon Sep 17 00:00:00 2001 From: Mondo Diaz Date: Thu, 11 Dec 2025 18:25:21 -0600 Subject: [PATCH] Enhance packages API with pagination, filtering, and metadata Backend changes: - Add format and platform columns to Package model with validation - Add database migrations for new columns with indexes - Create PackageDetailResponse schema with aggregated fields: - tag_count, artifact_count, total_size - latest_tag, latest_upload_at - recent_tags (last 5 tags) - Update list_packages endpoint: - Add pagination (page, limit) - Add search by name/description - Add sort (name, created_at, updated_at) and order (asc, desc) - Add format and platform filters - Return aggregated metadata for each package - Add GET /api/v1/project/{project}/packages/{package_name} endpoint - Returns single package with full metadata - Optional include_tags parameter for all tags - Update create_package to accept format and platform Frontend changes: - Update Package type with format, platform, and optional metadata fields - Update listPackages to handle paginated response --- backend/app/database.py | 26 +++++ backend/app/models.py | 12 +++ backend/app/routes.py | 217 +++++++++++++++++++++++++++++++++++++++- backend/app/schemas.py | 39 ++++++++ frontend/src/api.ts | 3 +- frontend/src/types.ts | 15 +++ 6 files changed, 306 insertions(+), 6 deletions(-) diff --git a/backend/app/database.py b/backend/app/database.py index 748f83d..c63119a 100644 --- a/backend/app/database.py +++ b/backend/app/database.py @@ -36,6 +36,32 @@ def _run_migrations(): END IF; END $$; """, + # Add format column to packages table + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'packages' AND column_name = 'format' + ) THEN + ALTER TABLE packages ADD COLUMN format VARCHAR(50) DEFAULT 'generic' NOT NULL; + CREATE INDEX IF NOT EXISTS idx_packages_format ON packages(format); + END IF; + END $$; + """, + # Add platform column to packages table + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'packages' AND column_name = 'platform' + ) THEN + ALTER TABLE packages ADD COLUMN platform VARCHAR(50) DEFAULT 'any' NOT NULL; + CREATE INDEX IF NOT EXISTS idx_packages_platform ON packages(platform); + END IF; + END $$; + """, ] with engine.connect() as conn: diff --git a/backend/app/models.py b/backend/app/models.py index 2081a32..491566b 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -38,6 +38,8 @@ class Package(Base): project_id = Column(UUID(as_uuid=True), ForeignKey("projects.id", ondelete="CASCADE"), nullable=False) name = Column(String(255), nullable=False) description = Column(Text) + format = Column(String(50), default="generic", nullable=False) + platform = Column(String(50), default="any", nullable=False) created_at = Column(DateTime(timezone=True), default=datetime.utcnow) updated_at = Column(DateTime(timezone=True), default=datetime.utcnow, onupdate=datetime.utcnow) @@ -49,6 +51,16 @@ class Package(Base): __table_args__ = ( Index("idx_packages_project_id", "project_id"), Index("idx_packages_name", "name"), + Index("idx_packages_format", "format"), + Index("idx_packages_platform", "platform"), + CheckConstraint( + "format IN ('generic', 'npm', 'pypi', 'docker', 'deb', 'rpm', 'maven', 'nuget', 'helm')", + name="check_package_format" + ), + CheckConstraint( + "platform IN ('any', 'linux', 'darwin', 'windows', 'linux-amd64', 'linux-arm64', 'darwin-amd64', 'darwin-arm64', 'windows-amd64')", + name="check_package_platform" + ), {"extend_existing": True}, ) diff --git a/backend/app/routes.py b/backend/app/routes.py index f2bf7d7..ab012a8 100644 --- a/backend/app/routes.py +++ b/backend/app/routes.py @@ -13,7 +13,8 @@ from .storage import get_storage, S3Storage, MULTIPART_CHUNK_SIZE from .models import Project, Package, Artifact, Tag, Upload, Consumer from .schemas import ( ProjectCreate, ProjectResponse, - PackageCreate, PackageResponse, + PackageCreate, PackageResponse, PackageDetailResponse, TagSummary, + PACKAGE_FORMATS, PACKAGE_PLATFORMS, ArtifactResponse, TagCreate, TagResponse, UploadResponse, @@ -119,14 +120,210 @@ def get_project(project_name: str, db: Session = Depends(get_db)): # Package routes -@router.get("/api/v1/project/{project_name}/packages", response_model=List[PackageResponse]) -def list_packages(project_name: str, db: Session = Depends(get_db)): +@router.get("/api/v1/project/{project_name}/packages", response_model=PaginatedResponse[PackageDetailResponse]) +def list_packages( + project_name: str, + page: int = Query(default=1, ge=1, description="Page number"), + limit: int = Query(default=20, ge=1, le=100, description="Items per page"), + search: Optional[str] = Query(default=None, description="Search by name or description"), + sort: str = Query(default="name", description="Sort field (name, created_at, updated_at)"), + order: str = Query(default="asc", description="Sort order (asc, desc)"), + format: Optional[str] = Query(default=None, description="Filter by package format"), + platform: Optional[str] = Query(default=None, description="Filter by platform"), + db: Session = Depends(get_db), +): project = db.query(Project).filter(Project.name == project_name).first() if not project: raise HTTPException(status_code=404, detail="Project not found") - packages = db.query(Package).filter(Package.project_id == project.id).order_by(Package.name).all() - return packages + # Validate sort field + valid_sort_fields = {"name": Package.name, "created_at": Package.created_at, "updated_at": Package.updated_at} + if sort not in valid_sort_fields: + raise HTTPException(status_code=400, detail=f"Invalid sort field. Must be one of: {', '.join(valid_sort_fields.keys())}") + + # Validate order + if order not in ("asc", "desc"): + raise HTTPException(status_code=400, detail="Invalid order. Must be 'asc' or 'desc'") + + # Validate format filter + if format and format not in PACKAGE_FORMATS: + raise HTTPException(status_code=400, detail=f"Invalid format. Must be one of: {', '.join(PACKAGE_FORMATS)}") + + # Validate platform filter + if platform and platform not in PACKAGE_PLATFORMS: + raise HTTPException(status_code=400, detail=f"Invalid platform. Must be one of: {', '.join(PACKAGE_PLATFORMS)}") + + # Base query + query = db.query(Package).filter(Package.project_id == project.id) + + # Apply search filter (case-insensitive on name and description) + if search: + search_lower = search.lower() + query = query.filter( + or_( + func.lower(Package.name).contains(search_lower), + func.lower(Package.description).contains(search_lower) + ) + ) + + # Apply format filter + if format: + query = query.filter(Package.format == format) + + # Apply platform filter + if platform: + query = query.filter(Package.platform == platform) + + # Get total count before pagination + total = query.count() + + # Apply sorting + sort_column = valid_sort_fields[sort] + if order == "desc": + query = query.order_by(sort_column.desc()) + else: + query = query.order_by(sort_column.asc()) + + # Apply pagination + offset = (page - 1) * limit + packages = query.offset(offset).limit(limit).all() + + # Calculate total pages + total_pages = math.ceil(total / limit) if total > 0 else 1 + + # Build detailed responses with aggregated data + detailed_packages = [] + for pkg in packages: + # Get tag count + tag_count = db.query(func.count(Tag.id)).filter(Tag.package_id == pkg.id).scalar() or 0 + + # Get unique artifact count and total size via uploads + artifact_stats = db.query( + func.count(func.distinct(Upload.artifact_id)), + func.coalesce(func.sum(Artifact.size), 0) + ).join(Artifact, Upload.artifact_id == Artifact.id).filter( + Upload.package_id == pkg.id + ).first() + artifact_count = artifact_stats[0] if artifact_stats else 0 + total_size = artifact_stats[1] if artifact_stats else 0 + + # Get latest tag + latest_tag_obj = db.query(Tag).filter( + Tag.package_id == pkg.id + ).order_by(Tag.created_at.desc()).first() + latest_tag = latest_tag_obj.name if latest_tag_obj else None + + # Get latest upload timestamp + latest_upload = db.query(func.max(Upload.uploaded_at)).filter( + Upload.package_id == pkg.id + ).scalar() + + # Get recent tags (limit 5) + recent_tags_objs = db.query(Tag).filter( + Tag.package_id == pkg.id + ).order_by(Tag.created_at.desc()).limit(5).all() + recent_tags = [ + TagSummary(name=t.name, artifact_id=t.artifact_id, created_at=t.created_at) + for t in recent_tags_objs + ] + + detailed_packages.append(PackageDetailResponse( + id=pkg.id, + project_id=pkg.project_id, + name=pkg.name, + description=pkg.description, + format=pkg.format, + platform=pkg.platform, + created_at=pkg.created_at, + updated_at=pkg.updated_at, + tag_count=tag_count, + artifact_count=artifact_count, + total_size=total_size, + latest_tag=latest_tag, + latest_upload_at=latest_upload, + recent_tags=recent_tags, + )) + + return PaginatedResponse( + items=detailed_packages, + pagination=PaginationMeta( + page=page, + limit=limit, + total=total, + total_pages=total_pages, + ), + ) + + +@router.get("/api/v1/project/{project_name}/packages/{package_name}", response_model=PackageDetailResponse) +def get_package( + project_name: str, + package_name: str, + include_tags: bool = Query(default=False, description="Include all tags (not just recent 5)"), + db: Session = Depends(get_db), +): + """Get a single package with full metadata""" + project = db.query(Project).filter(Project.name == project_name).first() + if not project: + raise HTTPException(status_code=404, detail="Project not found") + + pkg = db.query(Package).filter( + Package.project_id == project.id, + Package.name == package_name + ).first() + if not pkg: + raise HTTPException(status_code=404, detail="Package not found") + + # Get tag count + tag_count = db.query(func.count(Tag.id)).filter(Tag.package_id == pkg.id).scalar() or 0 + + # Get unique artifact count and total size via uploads + artifact_stats = db.query( + func.count(func.distinct(Upload.artifact_id)), + func.coalesce(func.sum(Artifact.size), 0) + ).join(Artifact, Upload.artifact_id == Artifact.id).filter( + Upload.package_id == pkg.id + ).first() + artifact_count = artifact_stats[0] if artifact_stats else 0 + total_size = artifact_stats[1] if artifact_stats else 0 + + # Get latest tag + latest_tag_obj = db.query(Tag).filter( + Tag.package_id == pkg.id + ).order_by(Tag.created_at.desc()).first() + latest_tag = latest_tag_obj.name if latest_tag_obj else None + + # Get latest upload timestamp + latest_upload = db.query(func.max(Upload.uploaded_at)).filter( + Upload.package_id == pkg.id + ).scalar() + + # Get tags (all if include_tags=true, else limit 5) + tags_query = db.query(Tag).filter(Tag.package_id == pkg.id).order_by(Tag.created_at.desc()) + if not include_tags: + tags_query = tags_query.limit(5) + tags_objs = tags_query.all() + recent_tags = [ + TagSummary(name=t.name, artifact_id=t.artifact_id, created_at=t.created_at) + for t in tags_objs + ] + + return PackageDetailResponse( + id=pkg.id, + project_id=pkg.project_id, + name=pkg.name, + description=pkg.description, + format=pkg.format, + platform=pkg.platform, + created_at=pkg.created_at, + updated_at=pkg.updated_at, + tag_count=tag_count, + artifact_count=artifact_count, + total_size=total_size, + latest_tag=latest_tag, + latest_upload_at=latest_upload, + recent_tags=recent_tags, + ) @router.post("/api/v1/project/{project_name}/packages", response_model=PackageResponse) @@ -135,6 +332,14 @@ def create_package(project_name: str, package: PackageCreate, db: Session = Depe if not project: raise HTTPException(status_code=404, detail="Project not found") + # Validate format + if package.format not in PACKAGE_FORMATS: + raise HTTPException(status_code=400, detail=f"Invalid format. Must be one of: {', '.join(PACKAGE_FORMATS)}") + + # Validate platform + if package.platform not in PACKAGE_PLATFORMS: + raise HTTPException(status_code=400, detail=f"Invalid platform. Must be one of: {', '.join(PACKAGE_PLATFORMS)}") + existing = db.query(Package).filter(Package.project_id == project.id, Package.name == package.name).first() if existing: raise HTTPException(status_code=400, detail="Package already exists in this project") @@ -143,6 +348,8 @@ def create_package(project_name: str, package: PackageCreate, db: Session = Depe project_id=project.id, name=package.name, description=package.description, + format=package.format, + platform=package.platform, ) db.add(db_package) db.commit() diff --git a/backend/app/schemas.py b/backend/app/schemas.py index 7b93ae5..837c0ce 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -39,10 +39,17 @@ class ProjectResponse(BaseModel): from_attributes = True +# Package format and platform enums +PACKAGE_FORMATS = ["generic", "npm", "pypi", "docker", "deb", "rpm", "maven", "nuget", "helm"] +PACKAGE_PLATFORMS = ["any", "linux", "darwin", "windows", "linux-amd64", "linux-arm64", "darwin-amd64", "darwin-arm64", "windows-amd64"] + + # Package schemas class PackageCreate(BaseModel): name: str description: Optional[str] = None + format: str = "generic" + platform: str = "any" class PackageResponse(BaseModel): @@ -50,6 +57,8 @@ class PackageResponse(BaseModel): project_id: UUID name: str description: Optional[str] + format: str + platform: str created_at: datetime updated_at: datetime @@ -57,6 +66,36 @@ class PackageResponse(BaseModel): from_attributes = True +class TagSummary(BaseModel): + """Lightweight tag info for embedding in package responses""" + name: str + artifact_id: str + created_at: datetime + + +class PackageDetailResponse(BaseModel): + """Package with aggregated metadata""" + id: UUID + project_id: UUID + name: str + description: Optional[str] + format: str + platform: str + created_at: datetime + updated_at: datetime + # Aggregated fields + tag_count: int = 0 + artifact_count: int = 0 + total_size: int = 0 + latest_tag: Optional[str] = None + latest_upload_at: Optional[datetime] = None + # Recent tags (limit 5) + recent_tags: List[TagSummary] = [] + + class Config: + from_attributes = True + + # Artifact schemas class ArtifactResponse(BaseModel): id: str diff --git a/frontend/src/api.ts b/frontend/src/api.ts index fe06ae1..bd876ce 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -45,7 +45,8 @@ export async function getProject(name: string): Promise { // Package API export async function listPackages(projectName: string): Promise { const response = await fetch(`${API_BASE}/project/${projectName}/packages`); - return handleResponse(response); + const data = await handleResponse>(response); + return data.items; } export async function createPackage(projectName: string, data: { name: string; description?: string }): Promise { diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 291ab87..444d80d 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -8,13 +8,28 @@ export interface Project { created_by: string; } +export interface TagSummary { + name: string; + artifact_id: string; + created_at: string; +} + export interface Package { id: string; project_id: string; name: string; description: string | null; + format: string; + platform: string; created_at: string; updated_at: string; + // Aggregated fields (from PackageDetailResponse) + tag_count?: number; + artifact_count?: number; + total_size?: number; + latest_tag?: string | null; + latest_upload_at?: string | null; + recent_tags?: TagSummary[]; } export interface Artifact {