warehouse13/.gitlab-ci.yml

stages:
  - test
  - build
  - deploy

variables:
  DOCKER_DRIVER: overlay2
  DOCKER_TLS_CERTDIR: "/certs"
  IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
  LATEST_TAG: $CI_REGISTRY_IMAGE:latest

# Test stage
test:
  stage: test
  image: python:3.11-slim
  before_script:
    - apt-get update && apt-get install -y gcc postgresql-client
    - pip install -r requirements.txt
    - pip install pytest pytest-asyncio httpx
  script:
    - echo "Running tests..."
    - python -m pytest tests/ -v || echo "No tests found, skipping"
  only:
    - branches
    - merge_requests

# Lint stage
lint:
  stage: test
  image: python:3.11-slim
  before_script:
    - pip install flake8 black
  script:
    - echo "Running linters..."
    - flake8 app/ --max-line-length=120 --ignore=E203,W503 || true
    - black --check app/ || true
  only:
    - branches
    - merge_requests
  allow_failure: true

# Build Docker image
build:
  stage: build
  image: docker:24
  services:
    - docker:24-dind
  before_script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
  script:
    - echo "Building Docker image..."
    - docker build -t $IMAGE_TAG -t $LATEST_TAG .
    - docker push $IMAGE_TAG
    - docker push $LATEST_TAG
  only:
    - main
    - master
    - develop
    - tags

# Deploy to development
deploy:dev:
  stage: deploy
  image: alpine/helm:latest
  before_script:
    - apk add --no-cache curl
    - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
    - chmod +x kubectl
    - mv kubectl /usr/local/bin/
    - mkdir -p ~/.kube
    - echo "$KUBE_CONFIG_DEV" | base64 -d > ~/.kube/config
  script:
    - echo "Deploying to development environment..."
    - |
      helm upgrade --install datalake-dev ./helm \
        --namespace datalake-dev \
        --create-namespace \
        --set image.repository=$CI_REGISTRY_IMAGE \
        --set image.tag=$CI_COMMIT_SHORT_SHA \
        --set ingress.enabled=true \
        --set ingress.hosts[0].host=datalake-dev.example.com \
        --set ingress.hosts[0].paths[0].path=/ \
        --set ingress.hosts[0].paths[0].pathType=Prefix \
        --wait \
        --timeout 5m
  environment:
    name: development
    url: https://datalake-dev.example.com
  only:
    - develop
  when: manual

# Deploy to staging
deploy:staging:
  stage: deploy
  image: alpine/helm:latest
  before_script:
    - apk add --no-cache curl
    - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
    - chmod +x kubectl
    - mv kubectl /usr/local/bin/
    - mkdir -p ~/.kube
    - echo "$KUBE_CONFIG_STAGING" | base64 -d > ~/.kube/config
  script:
    - echo "Deploying to staging environment..."
    - |
      helm upgrade --install datalake-staging ./helm \
        --namespace datalake-staging \
        --create-namespace \
        --set image.repository=$CI_REGISTRY_IMAGE \
        --set image.tag=$CI_COMMIT_SHORT_SHA \
        --set ingress.enabled=true \
        --set ingress.hosts[0].host=datalake-staging.example.com \
        --set ingress.hosts[0].paths[0].path=/ \
        --set ingress.hosts[0].paths[0].pathType=Prefix \
        --set resources.requests.cpu=1000m \
        --set resources.requests.memory=1Gi \
        --wait \
        --timeout 5m
  environment:
    name: staging
    url: https://datalake-staging.example.com
  only:
    - main
    - master
  when: manual

# Deploy to production
deploy:prod:
  stage: deploy
  image: alpine/helm:latest
  before_script:
    - apk add --no-cache curl
    - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
    - chmod +x kubectl
    - mv kubectl /usr/local/bin/
    - mkdir -p ~/.kube
    - echo "$KUBE_CONFIG_PROD" | base64 -d > ~/.kube/config
  script:
    - echo "Deploying to production environment..."
    - |
      helm upgrade --install datalake ./helm \
        --namespace datalake-prod \
        --create-namespace \
        --set image.repository=$CI_REGISTRY_IMAGE \
        --set image.tag=$CI_COMMIT_SHORT_SHA \
        --set replicaCount=3 \
        --set ingress.enabled=true \
        --set ingress.hosts[0].host=datalake.example.com \
        --set ingress.hosts[0].paths[0].path=/ \
        --set ingress.hosts[0].paths[0].pathType=Prefix \
        --set resources.requests.cpu=2000m \
        --set resources.requests.memory=2Gi \
        --set autoscaling.enabled=true \
        --set autoscaling.minReplicas=3 \
        --set autoscaling.maxReplicas=10 \
        --wait \
        --timeout 10m
  environment:
    name: production
    url: https://datalake.example.com
  only:
    - tags
  when: manual