Configure prod and stage for AWS services (RDS, S3, Secrets Manager)

.gitlab-ci.yml

@@ -336,7 +336,7 @@ cleanup_feature:
       when: manual
   allow_failure: true
 
-# Deploy to production (version tags only, manual approval required)
+# Deploy to production (version tags only)
 deploy_prod:
   stage: deploy
   # For tag pipelines, most jobs don't run (trusting main was tested)
@@ -371,7 +371,7 @@ deploy_prod:
   rules:
     # Only run on semantic version tags (v1.0.0, v1.2.3, etc.)
     - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+$/'
-      when: manual  # Require manual approval for prod
+      when: on_success
   allow_failure: false
 
 # Smoke tests for production deployment (read-only, no test data creation)

helm/orchard/templates/_helpers.tpl

@@ -77,6 +77,8 @@ PostgreSQL secret name
 {{- define "orchard.postgresql.secretName" -}}
 {{- if .Values.orchard.database.existingSecret }}
 {{- .Values.orchard.database.existingSecret }}
+{{- else if and .Values.orchard.database.secretsManager .Values.orchard.database.secretsManager.enabled }}
+{{- printf "%s-db-credentials" (include "orchard.fullname" .) }}
 {{- else if .Values.postgresql.enabled }}
 {{- printf "%s-postgresql" .Release.Name }}
 {{- else }}
@@ -90,6 +92,8 @@ PostgreSQL password key in secret
 {{- define "orchard.postgresql.passwordKey" -}}
 {{- if .Values.orchard.database.existingSecret -}}
 {{- .Values.orchard.database.existingSecretPasswordKey -}}
+{{- else if and .Values.orchard.database.secretsManager .Values.orchard.database.secretsManager.enabled -}}
+password
 {{- else if .Values.postgresql.enabled -}}
 password
 {{- else -}}

helm/orchard/templates/deployment.yaml

@@ -77,8 +77,16 @@ spec:
               value: {{ include "orchard.postgresql.host" . | quote }}
             - name: ORCHARD_DATABASE_PORT
               value: {{ .Values.orchard.database.port | quote }}
+            {{- if and .Values.orchard.database.secretsManager .Values.orchard.database.secretsManager.enabled }}
+            - name: ORCHARD_DATABASE_USER
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "orchard.postgresql.secretName" . }}
+                  key: username
+            {{- else }}
             - name: ORCHARD_DATABASE_USER
               value: {{ .Values.orchard.database.user | default .Values.postgresql.auth.username | quote }}
+            {{- end }}
             - name: ORCHARD_DATABASE_DBNAME
               value: {{ .Values.orchard.database.dbname | default .Values.postgresql.auth.database | quote }}
             - name: ORCHARD_DATABASE_SSLMODE
@@ -96,6 +104,7 @@ spec:
               value: {{ .Values.orchard.s3.bucket | quote }}
             - name: ORCHARD_S3_USE_PATH_STYLE
               value: {{ .Values.orchard.s3.usePathStyle | quote }}
+            {{- if or .Values.minio.enabled .Values.orchard.s3.existingSecret .Values.orchard.s3.accessKeyId }}
             - name: ORCHARD_S3_ACCESS_KEY_ID
               valueFrom:
                 secretKeyRef:
@@ -106,6 +115,7 @@ spec:
                 secretKeyRef:
                   name: {{ include "orchard.minio.secretName" . }}
                   key: {{ if .Values.minio.enabled }}root-password{{ else }}{{ .Values.orchard.s3.existingSecretSecretKeyKey }}{{ end }}
+            {{- end }}
             - name: ORCHARD_DOWNLOAD_MODE
               value: {{ .Values.orchard.download.mode | quote }}
             - name: ORCHARD_PRESIGNED_URL_EXPIRY
@@ -116,12 +126,27 @@ spec:
               value: {{ .Values.orchard.rateLimit.login | quote }}
             {{- end }}
             {{- end }}
+          {{- if and .Values.orchard.database.secretsManager .Values.orchard.database.secretsManager.enabled }}
+          volumeMounts:
+            - name: db-secrets
+              mountPath: /mnt/secrets-store
+              readOnly: true
+          {{- end }}
           livenessProbe:
             {{- toYaml .Values.livenessProbe | nindent 12 }}
           readinessProbe:
             {{- toYaml .Values.readinessProbe | nindent 12 }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+      {{- if and .Values.orchard.database.secretsManager .Values.orchard.database.secretsManager.enabled }}
+      volumes:
+        - name: db-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: {{ include "orchard.fullname" . }}-db-secret
+      {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

helm/orchard/templates/secret-provider-class.yaml

@@ -0,0 +1,27 @@
+{{- if and .Values.orchard.database.secretsManager .Values.orchard.database.secretsManager.enabled }}
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: {{ include "orchard.fullname" . }}-db-secret
+  labels:
+    {{- include "orchard.labels" . | nindent 4 }}
+spec:
+  provider: aws
+  parameters:
+    objects: |
+      - objectName: "{{ .Values.orchard.database.secretsManager.secretArn }}"
+        objectType: "secretsmanager"
+        jmesPath:
+          - path: username
+            objectAlias: db-username
+          - path: password
+            objectAlias: db-password
+  secretObjects:
+    - secretName: {{ include "orchard.fullname" . }}-db-credentials
+      type: Opaque
+      data:
+        - objectName: db-username
+          key: username
+        - objectName: db-password
+          key: password
+{{- end }}

helm/orchard/values-prod.yaml

@@ -4,7 +4,7 @@ replicaCount: 1
 
 image:
   repository: registry.global.bsf.tools/esv/bsf/bsf-integration/orchard/orchard-mvp
-  pullPolicy: IfNotPresent  # Don't always pull in prod
+  pullPolicy: Always
   tag: "latest"  # Overridden by CI
 
 imagePullSecrets:
@@ -19,7 +19,8 @@ initContainer:
 serviceAccount:
   create: true
   automount: true
-  annotations: {}
+  annotations:
+    eks.amazonaws.com/role-arn: arn:aws-us-gov:iam::052673043337:role/service-orchard-prd
   name: "orchard"
 
 podAnnotations: {}
@@ -41,6 +42,7 @@ ingress:
   className: "nginx"
   annotations:
     cert-manager.io/cluster-issuer: "letsencrypt"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"  # Disable body size limit for uploads
   hosts:
     - host: orchard.common.global.bsf.tools
       paths:
@@ -90,122 +92,38 @@ orchard:
     host: "0.0.0.0"
     port: 8080
 
-  # Database configuration (used when postgresql.enabled is false)
+  # Database configuration - uses AWS Secrets Manager via CSI driver
-  # TODO: Configure for managed PostgreSQL when ready
   database:
-    host: ""
+    host: "orchard-prd.cluster-cvw3jzjkozoc.us-gov-west-1.rds.amazonaws.com"
     port: 5432
-    user: orchard
+    dbname: orchard_prod
-    password: ""
+    sslmode: require
-    dbname: orchard
+    secretsManager:
-    sslmode: disable
+      enabled: true
-    existingSecret: ""
+      secretArn: "arn:aws-us-gov:secretsmanager:us-gov-west-1:052673043337:secret:rds!cluster-0afc8af5-f644-4284-92fb-2ed545490f92-3v9uXV"
-    existingSecretPasswordKey: "password"
 
-  # S3 configuration (used when minio.enabled is false)
+  # S3 configuration - uses IRSA for credentials
-  # TODO: Configure for real S3 when ready
   s3:
-    endpoint: ""
+    endpoint: ""  # Empty = use AWS default
-    region: us-east-1
+    region: us-gov-west-1
-    bucket: orchard-artifacts
+    bucket: orchard-artifacts-prod
-    accessKeyId: ""
+    usePathStyle: false  # Real S3 uses virtual-hosted style
-    secretAccessKey: ""
-    usePathStyle: true
-    existingSecret: ""
-    existingSecretAccessKeyKey: "access-key-id"
-    existingSecretSecretKeyKey: "secret-access-key"
 
   download:
     mode: "presigned"
     presignedUrlExpiry: 3600
 
-# PostgreSQL subchart - MVP uses subchart, switch to managed later
+# PostgreSQL subchart - disabled in prod, using RDS
 postgresql:
-  enabled: true
+  enabled: false
-  image:
-    registry: containers.global.bsf.tools
-    repository: bitnami/postgresql
-    tag: "15"
-    pullPolicy: IfNotPresent
-  auth:
-    username: orchard
-    password: orchard-prod-password  # TODO: Use existingSecret
-    database: orchard
-  primary:
-    persistence:
-      enabled: true  # Enable persistence for prod
-      size: 20Gi
-    resourcesPreset: "none"
-    resources:
-      limits:
-        cpu: 500m
-        memory: 512Mi
-      requests:
-        cpu: 250m
-        memory: 512Mi
-  volumePermissions:
-    resourcesPreset: "none"
-    resources:
-      limits:
-        cpu: 100m
-        memory: 128Mi
-      requests:
-        cpu: 50m
-        memory: 128Mi
 
-# MinIO subchart - MVP uses subchart, switch to real S3 later
+# MinIO subchart - disabled in prod, using real S3
 minio:
-  enabled: true
+  enabled: false
-  image:
-    registry: containers.global.bsf.tools
-    repository: bitnami/minio
-    tag: "latest"
-    pullPolicy: IfNotPresent
-  auth:
-    rootUser: minioadmin
-    rootPassword: minioadmin-prod  # TODO: Use existingSecret
-  defaultBuckets: "orchard-artifacts"
-  persistence:
-    enabled: true  # Enable persistence for prod
-    size: 100Gi
-  resourcesPreset: "none"
-  resources:
-    limits:
-      cpu: 500m
-      memory: 512Mi
-    requests:
-      cpu: 250m
-      memory: 512Mi
-  defaultInitContainers:
-    volumePermissions:
-      resourcesPreset: "none"
-      resources:
-        limits:
-          cpu: 100m
-          memory: 128Mi
-        requests:
-          cpu: 50m
-          memory: 128Mi
-  provisioning:
-    resources:
-      limits:
-        cpu: 200m
-        memory: 256Mi
-      requests:
-        cpu: 100m
-        memory: 256Mi
 
-# MinIO external ingress for presigned URL access
+# MinIO ingress - disabled in prod, using real S3
 minioIngress:
-  enabled: true
+  enabled: false
-  className: "nginx"
-  annotations:
-    cert-manager.io/cluster-issuer: "letsencrypt"
-    nginx.ingress.kubernetes.io/proxy-body-size: "0"
-  host: "minio-orchard.common.global.bsf.tools"
-  tls:
-    enabled: true
-    secretName: minio-prod-tls
 
 redis:
   enabled: false

helm/orchard/values-stage.yaml

@@ -19,7 +19,8 @@ initContainer:
 serviceAccount:
   create: true
   automount: true
-  annotations: {}
+  annotations:
+    eks.amazonaws.com/role-arn: arn:aws-us-gov:iam::052673043337:role/service-orchard-stage
   name: "orchard"
 
 podAnnotations: {}
@@ -93,28 +94,22 @@ orchard:
     host: "0.0.0.0"
     port: 8080
 
-  # Database configuration (used when postgresql.enabled is false)
+  # Database configuration - uses AWS Secrets Manager via CSI driver
   database:
-    host: ""
+    host: "orchard-stage.cluster-cvw3jzjkozoc.us-gov-west-1.rds.amazonaws.com"
     port: 5432
-    user: orchard
+    dbname: postgres
-    password: ""
+    sslmode: require
-    dbname: orchard
+    secretsManager:
-    sslmode: disable
+      enabled: true
-    existingSecret: ""
+      secretArn: "arn:aws-us-gov:secretsmanager:us-gov-west-1:052673043337:secret:rds!cluster-a573672b-1a38-4665-a654-1b7df37b5297-IaeFQL"
-    existingSecretPasswordKey: "password"
 
-  # S3 configuration (used when minio.enabled is false)
+  # S3 configuration - uses IRSA for credentials
   s3:
-    endpoint: ""
+    endpoint: ""  # Empty = use AWS default
-    region: us-east-1
+    region: us-gov-west-1
-    bucket: orchard-artifacts
+    bucket: orchard-artifacts-stage
-    accessKeyId: ""
+    usePathStyle: false  # Real S3 uses virtual-hosted style
-    secretAccessKey: ""
-    usePathStyle: true
-    existingSecret: ""
-    existingSecretAccessKeyKey: "access-key-id"
-    existingSecretSecretKeyKey: "secret-access-key"
 
   # Download configuration
   download:
@@ -125,98 +120,17 @@ orchard:
   rateLimit:
     login: "1000/minute"  # Default is 5/minute, relaxed for CI integration tests
 
-# PostgreSQL subchart configuration
+# PostgreSQL subchart - disabled in stage, using RDS
 postgresql:
-  enabled: true
-  image:
-    registry: containers.global.bsf.tools
-    repository: bitnami/postgresql
-    tag: "15"
-    pullPolicy: IfNotPresent
-  auth:
-    username: orchard
-    password: orchard-password
-    database: orchard
-  primary:
-    persistence:
   enabled: false
-      size: 10Gi
-    # Resources with memory requests = limits per cluster policy
-    resourcesPreset: "none"
-    resources:
-      limits:
-        cpu: 500m
-        memory: 512Mi
-      requests:
-        cpu: 250m
-        memory: 512Mi
-  # Volume permissions init container
-  volumePermissions:
-    resourcesPreset: "none"
-    resources:
-      limits:
-        cpu: 100m
-        memory: 128Mi
-      requests:
-        cpu: 50m
-        memory: 128Mi
 
-# MinIO subchart configuration
+# MinIO subchart - disabled in stage, using real S3
 minio:
-  enabled: true
-  image:
-    registry: containers.global.bsf.tools
-    repository: bitnami/minio
-    tag: "latest"
-    pullPolicy: IfNotPresent
-  auth:
-    rootUser: minioadmin
-    rootPassword: minioadmin
-  defaultBuckets: "orchard-artifacts"
-  persistence:
   enabled: false
-    size: 50Gi
-  # Resources with memory requests = limits per cluster policy
-  resourcesPreset: "none"  # Disable preset to use explicit resources
-  resources:
-    limits:
-      cpu: 500m
-      memory: 512Mi
-    requests:
-      cpu: 250m
-      memory: 512Mi
-  # Init container resources
-  defaultInitContainers:
-    volumePermissions:
-      resourcesPreset: "none"
-      resources:
-        limits:
-          cpu: 100m
-          memory: 128Mi
-        requests:
-          cpu: 50m
-          memory: 128Mi
-  # Provisioning job resources
-  provisioning:
-    resources:
-      limits:
-        cpu: 200m
-        memory: 256Mi
-      requests:
-        cpu: 100m
-        memory: 256Mi
 
-# MinIO external ingress for presigned URL access (separate from subchart ingress)
+# MinIO ingress - disabled in stage, using real S3
 minioIngress:
-  enabled: true
+  enabled: false
-  className: "nginx"
-  annotations:
-    cert-manager.io/cluster-issuer: "letsencrypt"
-    nginx.ingress.kubernetes.io/proxy-body-size: "0"  # Disable body size limit for uploads
-  host: "minio-orch-stage.common.global.bsf.tools"
-  tls:
-    enabled: true
-    secretName: minio-tls
 
 # Redis subchart configuration (for future caching)
 redis: