Compare commits

...

10 Commits

Author SHA1 Message Date
Armando Diaz
e08ab62a32 chart and ci tweaks 2025-10-17 08:13:54 -05:00
Armando Diaz
c7ae399615 fix values file location 2025-10-17 08:01:55 -05:00
33d06bc94d Clean up Helm directory and update documentation
- Removed deprecated chart files from helm/ root directory
- Updated all Helm documentation to reference warehouse13 chart
- Changed database name from 'datalake' to 'warehouse13' in values.yaml
- Updated helm command examples in SUMMARY.md
- Fixed migration instructions in helm/README.md
- Updated PostgreSQL backup/restore commands with correct database name

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-10-17 08:01:05 -05:00
e2e5c683e4 Fix Helm template error: update _helpers.tpl to use app instead of api
- Changed .Values.api.env.databaseUrl to .Values.app.env.databaseUrl
- This aligns with the unified architecture where api and frontend are combined into a single app
- Chart now passes helm lint successfully

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-10-17 07:55:51 -05:00
Armando Diaz
838e145598 fix release name 2025-10-16 21:20:15 -05:00
15e0f886d7 Merge pull request 'main' (#4) from main into pipeline
Reviewed-on: mondo/SIM-Data-Platform#4
2025-10-16 21:14:55 -05:00
7126c618ea Merge pull request 'f/npm' (#3) from f/npm into main
Reviewed-on: mondo/SIM-Data-Platform#3
2025-10-16 15:49:00 -05:00
Patel (US), Pratik
543617cc08 Merge remote-tracking branch 'origin/pipeline' into f/npm 2025-10-16 15:17:24 -05:00
Patel (US), Pratik
090361cf66 test npm changes 2025-10-16 14:44:08 -05:00
pratik
18e70cd445 Toggle NPM througn env file 2025-10-16 14:25:08 -05:00
17 changed files with 45 additions and 438 deletions

View File

@@ -29,6 +29,7 @@ API_PORT=8000
MAX_UPLOAD_SIZE=524288000
# NPM Configuration (for frontend build)
# Leave blank or set to https://registry.npmjs.org/ for default npm registry
# Set to your custom npm proxy/registry URL if needed (e.g., http://your-nexus-server:8081/repository/npm-proxy/)
NPM_REGISTRY=
# Default: https://registry.npmjs.org/ (public npm registry)
# For restricted environments, set to your custom npm proxy/registry URL
# Example: http://your-nexus-server:8081/repository/npm-proxy/
NPM_REGISTRY=https://registry.npmjs.org/

View File

@@ -1,19 +1,7 @@
stages:
- test
- build
- deploy
# Test stage
test:
stage: test
allow_failure: true
image: containers.global.bsf.tools/node:20.11-alpine3.19
script:
- cd frontend
- npm config set registry https://deps.global.bsf.tools/artifactory/api/npm/registry.npmjs.org/
- npm config set strict-ssl false
- npm config fix
- npm install
build_container:
stage: build
@@ -40,14 +28,14 @@ deploy_helm_charts:
# NAMESPACE: "bsf-services-namespace"
# ONLY: "main"
- ENV: "dev"
VALUES_FILE: "helm/values.yaml"
VALUES_FILE: "helm/warehouse13/values.yaml"
CONTEXT: "esv/bsf/bsf-services/gitlab-kaas-agent-config:services-prod-agent"
NAMESPACE: "bsf-services-dev-namespace"
# ONLY: ["branches", "!main"]
script:
- kubectl config use-context $CONTEXT
- |
helm upgrade --install warehouse13-$CI_COMMIT_REF \
helm upgrade --install warehouse13-$CI_COMMIT_REF_NAME \
./helm/warehouse13 --namespace $NAMESPACE \
-f $VALUES_FILE \
--set api.image=$CI_REGISTRY_IMAGE \

View File

@@ -38,7 +38,10 @@ services:
app:
container_name: warehouse13-app
build: .
build:
context: .
args:
NPM_REGISTRY: ${NPM_REGISTRY:-https://registry.npmjs.org/}
ports:
- "8000:8000"
environment:

View File

@@ -164,7 +164,7 @@ curl -X POST "http://localhost:8000/api/v1/artifacts/query" \
make deploy
# Or directly with Helm
helm install datalake ./helm --namespace datalake --create-namespace
helm install warehouse13 ./helm/warehouse13 --namespace warehouse13 --create-namespace
```
## Feature Flags Usage
@@ -190,9 +190,8 @@ AWS_REGION=us-east-1
S3_BUCKET_NAME=your-bucket
# Deploy
helm install datalake ./helm \
--set config.deploymentMode=cloud \
--set aws.enabled=true
helm install warehouse13 ./helm/warehouse13 \
--set global.deploymentMode=cloud
```
## What's Next

View File

@@ -23,20 +23,20 @@
},
"private": true,
"dependencies": {
"@angular/common": "^19.1.0",
"@angular/compiler": "^19.1.0",
"@angular/core": "^19.1.0",
"@angular/forms": "^19.1.0",
"@angular/platform-browser": "^19.1.0",
"@angular/router": "^19.1.0",
"@angular/common": "19.2.x",
"@angular/compiler": "19.2.x",
"@angular/core": "19.2.x",
"@angular/forms": "19.2.x",
"@angular/platform-browser": "19.2.x",
"@angular/router": "19.2.x",
"rxjs": "~7.8.0",
"tslib": "^2.8.1",
"zone.js": "~0.15.0"
},
"devDependencies": {
"@angular/build": "<=19.2.7",
"@angular/cli": "<=19.2.7",
"@angular/compiler-cli": "^19.1.0",
"@angular/build": "19.2.x",
"@angular/cli": "19.2.x",
"@angular/compiler-cli": "19.2.x",
"@types/jasmine": "~5.1.0",
"jasmine-core": "~5.9.0",
"karma": "~6.4.0",
@@ -44,7 +44,7 @@
"karma-coverage": "~2.2.0",
"karma-jasmine": "~5.1.0",
"karma-jasmine-html-reporter": "~2.1.0",
"typescript": "~5.8.0",
"typescript": "5.x.x",
"undici-types": "7.12.0",
"node-releases": "2.0.21",
"node-gyp": "11.4.2",

View File

@@ -1,14 +0,0 @@
apiVersion: v2
name: warehouse13
description: Warehouse13 - Enterprise Test Artifact Storage (Legacy Chart - Use ./warehouse13 instead)
type: application
version: 1.0.0
appVersion: "1.0.0"
keywords:
- testing
- artifacts
- storage
- datalake
deprecated: true
maintainers:
- name: Warehouse13 Team

View File

@@ -20,23 +20,19 @@ helm install warehouse13 ./warehouse13
**Documentation:** See [warehouse13/README.md](./warehouse13/README.md)
## Legacy Chart (Deprecated)
## Migration from Legacy Chart
The files in this root `helm/` directory are from an older version and are marked as deprecated. Please use the `./warehouse13/` chart instead.
## Migration
If you're using the old chart, migration is straightforward:
If you were using an older version of the chart, migration is straightforward:
```bash
# Uninstall old chart
helm uninstall datalake
# Uninstall old chart (if named "datalake" or other name)
helm uninstall <old-release-name>
# Install new chart
helm install warehouse13 ./warehouse13
helm install warehouse13 ./warehouse13 --namespace warehouse13 --create-namespace
# Or upgrade in place (if compatible)
helm upgrade datalake ./warehouse13
helm upgrade <old-release-name> ./warehouse13
```
Note: Check your values.yaml configuration and update image repositories, resource limits, and other settings as needed.

View File

@@ -1,60 +0,0 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "w13.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
*/}}
{{- define "w13.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "w13.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "w13.labels" -}}
helm.sh/chart: {{ include "w13.chart" . }}
{{ include "w13.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "w13.selectorLabels" -}}
app.kubernetes.io/name: {{ include "w13.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "w13.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "w13.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}

View File

@@ -1,111 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "w13.fullname" . }}
labels:
{{- include "w13.labels" . | nindent 4 }}
spec:
{{- if not .Values.autoscaling.enabled }}
replicas: {{ .Values.replicaCount }}
{{- end }}
selector:
matchLabels:
{{- include "w13.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "w13.selectorLabels" . | nindent 8 }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "w13.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
containers:
- name: {{ .Chart.Name }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- name: http
containerPort: {{ .Values.service.targetPort }}
protocol: TCP
livenessProbe:
httpGet:
path: /health
port: http
initialDelaySeconds: 30
periodSeconds: 10
readinessProbe:
httpGet:
path: /health
port: http
initialDelaySeconds: 5
periodSeconds: 5
env:
- name: DATABASE_URL
valueFrom:
secretKeyRef:
name: {{ include "w13.fullname" . }}-secrets
key: database-url
- name: STORAGE_BACKEND
value: {{ .Values.config.storageBackend | quote }}
- name: MAX_UPLOAD_SIZE
value: {{ .Values.config.maxUploadSize | quote }}
{{- if eq .Values.config.storageBackend "s3" }}
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: {{ include "w13.fullname" . }}-secrets
key: aws-access-key-id
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ include "w13.fullname" . }}-secrets
key: aws-secret-access-key
- name: AWS_REGION
value: {{ .Values.aws.region | quote }}
- name: S3_BUCKET_NAME
value: {{ .Values.aws.bucketName | quote }}
{{- else }}
- name: MINIO_ENDPOINT
value: "{{ include "w13.fullname" . }}-minio:9000"
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ include "w13.fullname" . }}-secrets
key: minio-access-key
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: {{ include "w13.fullname" . }}-secrets
key: minio-secret-key
- name: MINIO_BUCKET_NAME
value: "test-artifacts"
- name: MINIO_SECURE
value: "false"
{{- end }}
{{- with .Values.env }}
{{- toYaml . | nindent 8 }}
{{- end }}
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}

View File

@@ -1,41 +0,0 @@
{{- if .Values.ingress.enabled -}}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {{ include "datalake.fullname" . }}
labels:
{{- include "datalake.labels" . | nindent 4 }}
{{- with .Values.ingress.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if .Values.ingress.className }}
ingressClassName: {{ .Values.ingress.className }}
{{- end }}
{{- if .Values.ingress.tls }}
tls:
{{- range .Values.ingress.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- range .Values.ingress.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- range .paths }}
- path: {{ .path }}
pathType: {{ .pathType }}
backend:
service:
name: {{ include "datalake.fullname" $ }}
port:
number: {{ $.Values.service.port }}
{{- end }}
{{- end }}
{{- end }}

View File

@@ -1,16 +0,0 @@
apiVersion: v1
kind: Secret
metadata:
name: {{ include "datalake.fullname" . }}-secrets
labels:
{{- include "datalake.labels" . | nindent 4 }}
type: Opaque
stringData:
database-url: "postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ include "datalake.fullname" . }}-postgresql:5432/{{ .Values.postgresql.auth.database }}"
{{- if .Values.aws.enabled }}
aws-access-key-id: {{ .Values.aws.accessKeyId | quote }}
aws-secret-access-key: {{ .Values.aws.secretAccessKey | quote }}
{{- else }}
minio-access-key: {{ .Values.minio.rootUser | quote }}
minio-secret-key: {{ .Values.minio.rootPassword | quote }}
{{- end }}

View File

@@ -1,15 +0,0 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "datalake.fullname" . }}
labels:
{{- include "datalake.labels" . | nindent 4 }}
spec:
type: {{ .Values.service.type }}
ports:
- port: {{ .Values.service.port }}
targetPort: http
protocol: TCP
name: http
selector:
{{- include "datalake.selectorLabels" . | nindent 4 }}

View File

@@ -1,12 +0,0 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "datalake.serviceAccountName" . }}
labels:
{{- include "datalake.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

View File

@@ -1,111 +0,0 @@
replicaCount: 1
image:
repository: w13
pullPolicy: IfNotPresent
tag: "latest"
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
serviceAccount:
create: true
annotations: {}
name: ""
podAnnotations: {}
podSecurityContext:
fsGroup: 1000
securityContext:
capabilities:
drop:
- ALL
readOnlyRootFilesystem: false
runAsNonRoot: true
runAsUser: 1000
service:
type: ClusterIP
port: 8000
targetPort: 8000
ingress:
enabled: false
className: ""
annotations: {}
hosts:
- host: w13.local
paths:
- path: /
pathType: Prefix
tls: []
resources:
limits:
cpu: 1000m
memory: 1Gi
requests:
cpu: 500m
memory: 512Mi
autoscaling:
enabled: false
minReplicas: 1
maxReplicas: 10
targetCPUUtilizationPercentage: 80
nodeSelector: {}
tolerations: []
affinity: {}
# Application configuration
config:
storageBackend: minio # or "s3"
maxUploadSize: 524288000 # 500MB
# PostgreSQL configuration
postgresql:
enabled: true
auth:
username: user
password: password
database: w13
primary:
persistence:
enabled: true
size: 10Gi
# MinIO configuration (for self-hosted storage)
minio:
enabled: true
mode: standalone
rootUser: minioadmin
rootPassword: minioadmin
persistence:
enabled: true
size: 50Gi
service:
type: ClusterIP
port: 9000
consoleService:
port: 9001
# AWS S3 configuration (when using AWS)
aws:
enabled: false
accessKeyId: ""
secretAccessKey: ""
region: us-east-1
bucketName: test-artifacts
# Environment variables
env:
- name: API_HOST
value: "0.0.0.0"
- name: API_PORT
value: "8000"

View File

@@ -328,10 +328,10 @@ kubectl delete pvc -l app.kubernetes.io/instance=my-warehouse13
```bash
# Create backup
kubectl exec -it warehouse13-postgres-0 -- pg_dump -U user datalake > backup.sql
kubectl exec -it warehouse13-postgres-0 -- pg_dump -U user warehouse13 > backup.sql
# Restore
kubectl exec -i warehouse13-postgres-0 -- psql -U user datalake < backup.sql
kubectl exec -i warehouse13-postgres-0 -- psql -U user warehouse13 < backup.sql
```
### MinIO Backup

View File

@@ -63,8 +63,8 @@ Create the name of the service account to use
PostgreSQL connection string
*/}}
{{- define "warehouse13.postgresUrl" -}}
{{- if .Values.api.env.databaseUrl }}
{{- .Values.api.env.databaseUrl }}
{{- if .Values.app.env.databaseUrl }}
{{- .Values.app.env.databaseUrl }}
{{- else }}
{{- printf "postgresql://%s:%s@warehouse13-postgres:%d/%s" .Values.postgres.auth.username .Values.postgres.auth.password (.Values.postgres.service.port | int) .Values.postgres.auth.database }}
{{- end }}

View File

@@ -16,14 +16,14 @@ postgres:
auth:
username: user
password: password
database: datalake
database: warehouse13
persistence:
enabled: true
size: 10Gi
storageClass: ""
resources:
requests:
memory: "256Mi"
memory: "512Mi"
cpu: "250m"
limits:
memory: "512Mi"
@@ -48,7 +48,7 @@ minio:
storageClass: ""
resources:
requests:
memory: "512Mi"
memory: "1Gi"
cpu: "250m"
limits:
memory: "1Gi"
@@ -70,12 +70,12 @@ app:
pullPolicy: always
replicas: 2
env:
databaseUrl: "postgresql://user:password@warehouse13-postgres:5432/datalake"
databaseUrl: "postgresql://user:password@warehouse13-postgres:5432/warehouse13"
storageBackend: "minio"
minioEndpoint: "warehouse13-minio:9000"
resources:
requests:
memory: "384Mi"
memory: "768Mi"
cpu: "350m"
limits:
memory: "768Mi"
@@ -96,20 +96,20 @@ app:
# Ingress
ingress:
enabled: false
enabled: true
className: "nginx"
annotations:
# cert-manager.io/cluster-issuer: "letsencrypt-prod"
cert-manager.io/cluster-issuer: "letsencrypt"
hosts:
- host: warehouse13.example.com
- host: warehouse13.common.global.bsf.tools
paths:
- path: /
pathType: Prefix
backend: app # All traffic goes to unified app (serves both API and frontend)
tls: []
# - secretName: warehouse13-tls
# hosts:
# - warehouse13.example.com
tls:
- secretName: warehouse13-tls
hosts:
- warehouse13.common.global.bsf.tools
# Service Account
serviceAccount: