bitforge/orchard
3
0
Fork 0
Code Projects Wiki Activity

Add upstream caching infrastructure and refactor CI pipeline

Browse Source 
Upstream Caching (Epic #68-#75, #105):
- Add upstream_sources and cache_settings tables with migrations
- Add cache management API endpoints (CRUD for sources, settings)
- Add environment variable overrides for upstream sources and cache settings
- Add encryption module for storing credentials securely
- Add frontend Admin Cache Management page
- Add is_system field to projects for system cache distinction
- Add purge_seed_data for transitioning to production-like environments

CI Pipeline Refactoring:
- Remove reset jobs (reset_stage_pre, reset_stage)
- Add ephemeral orchard-test deployment for main branch testing
- Run integration tests on ephemeral deployment before promoting to stage
- Stage is now long-running pre-prod (smoke tests only)
- Disable prosper_setup for tag pipelines
This commit is contained in:
Mondo Diaz
2026-01-29 11:28:59 -06:00
parent c92895ffe9
commit a3a49ac9c3
24 changed files with 7271 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
backend/app/database.py
View File

@@ -9,6 +9,7 @@ import hashlib
from .config import get_settings
from .models import Base
from .purge_seed_data import should_purge_seed_data, purge_seed_data
settings = get_settings()
logger = logging.getLogger(__name__)
@@ -80,6 +81,14 @@ def init_db():
# Run migrations for schema updates
_run_migrations()
# Purge seed data if requested (for transitioning to production-like environment)
if should_purge_seed_data():
db = SessionLocal()
try:
purge_seed_data(db)
finally:
db.close()
def _ensure_migrations_table(conn) -> None:
"""Create the migrations tracking table if it doesn't exist."""
@@ -429,6 +438,99 @@ def _run_migrations():
END $$;
""",
),
Migration(
name="016_add_is_system_to_projects",
sql="""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = 'projects' AND column_name = 'is_system'
) THEN
ALTER TABLE projects ADD COLUMN is_system BOOLEAN NOT NULL DEFAULT FALSE;
CREATE INDEX IF NOT EXISTS idx_projects_is_system ON projects(is_system);
END IF;
END $$;
""",
),
Migration(
name="017_create_upstream_sources",
sql="""
CREATE TABLE IF NOT EXISTS upstream_sources (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
name VARCHAR(255) NOT NULL UNIQUE,
source_type VARCHAR(50) NOT NULL DEFAULT 'generic',
url VARCHAR(2048) NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT FALSE,
is_public BOOLEAN NOT NULL DEFAULT TRUE,
auth_type VARCHAR(20) NOT NULL DEFAULT 'none',
username VARCHAR(255),
password_encrypted BYTEA,
headers_encrypted BYTEA,
priority INTEGER NOT NULL DEFAULT 100,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
CONSTRAINT check_source_type CHECK (
source_type IN ('npm', 'pypi', 'maven', 'docker', 'helm', 'nuget', 'deb', 'rpm', 'generic')
),
CONSTRAINT check_auth_type CHECK (
auth_type IN ('none', 'basic', 'bearer', 'api_key')
),
CONSTRAINT check_priority_positive CHECK (priority > 0)
);
CREATE INDEX IF NOT EXISTS idx_upstream_sources_enabled ON upstream_sources(enabled);
CREATE INDEX IF NOT EXISTS idx_upstream_sources_source_type ON upstream_sources(source_type);
CREATE INDEX IF NOT EXISTS idx_upstream_sources_is_public ON upstream_sources(is_public);
CREATE INDEX IF NOT EXISTS idx_upstream_sources_priority ON upstream_sources(priority);
""",
),
Migration(
name="018_create_cache_settings",
sql="""
CREATE TABLE IF NOT EXISTS cache_settings (
id INTEGER PRIMARY KEY DEFAULT 1,
allow_public_internet BOOLEAN NOT NULL DEFAULT TRUE,
auto_create_system_projects BOOLEAN NOT NULL DEFAULT TRUE,
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
CONSTRAINT check_cache_settings_singleton CHECK (id = 1)
);
INSERT INTO cache_settings (id, allow_public_internet, auto_create_system_projects)
VALUES (1, TRUE, TRUE)
ON CONFLICT (id) DO NOTHING;
""",
),
Migration(
name="019_create_cached_urls",
sql="""
CREATE TABLE IF NOT EXISTS cached_urls (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
url VARCHAR(4096) NOT NULL,
url_hash VARCHAR(64) NOT NULL UNIQUE,
artifact_id VARCHAR(64) NOT NULL REFERENCES artifacts(id),
source_id UUID REFERENCES upstream_sources(id) ON DELETE SET NULL,
fetched_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
response_headers JSONB DEFAULT '{}',
created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_cached_urls_url_hash ON cached_urls(url_hash);
CREATE INDEX IF NOT EXISTS idx_cached_urls_artifact_id ON cached_urls(artifact_id);
CREATE INDEX IF NOT EXISTS idx_cached_urls_source_id ON cached_urls(source_id);
CREATE INDEX IF NOT EXISTS idx_cached_urls_fetched_at ON cached_urls(fetched_at);
""",
),
Migration(
name="020_seed_default_upstream_sources",
sql="""
INSERT INTO upstream_sources (id, name, source_type, url, enabled, is_public, auth_type, priority)
VALUES
(gen_random_uuid(), 'npm-public', 'npm', 'https://registry.npmjs.org', FALSE, TRUE, 'none', 100),
(gen_random_uuid(), 'pypi-public', 'pypi', 'https://pypi.org/simple', FALSE, TRUE, 'none', 100),
(gen_random_uuid(), 'maven-central', 'maven', 'https://repo1.maven.org/maven2', FALSE, TRUE, 'none', 100),
(gen_random_uuid(), 'docker-hub', 'docker', 'https://registry-1.docker.io', FALSE, TRUE, 'none', 100)
ON CONFLICT (name) DO NOTHING;
""",
),
]
with engine.connect() as conn: