diff --git a/infra/litellm/Chart.yaml b/infra/litellm/Chart.yaml index 14170b9..5edfc47 100644 --- a/infra/litellm/Chart.yaml +++ b/infra/litellm/Chart.yaml @@ -1,12 +1,16 @@ apiVersion: v2 -name: litellm -description: A Helm chart for Kubernetes +name: litellm-helm +description: Call all LLM APIs using the OpenAI format type: application - -version: 0.1.0 -appVersion: "1.16.0" +version: 0.4.1 +appVersion: v1.50.2 dependencies: - - name: "litellm-helm" - version: 0.1.636 - repository: oci://ghcr.io/berriai + - name: "postgresql" + version: ">=13.3.0" + repository: oci://registry-1.docker.io/bitnamicharts + condition: db.deployStandalone + - name: redis + version: ">=18.0.0" + repository: oci://registry-1.docker.io/bitnamicharts + condition: redis.enabled diff --git a/infra/litellm/templates/NOTES.txt b/infra/litellm/templates/NOTES.txt new file mode 100644 index 0000000..e72c991 --- /dev/null +++ b/infra/litellm/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "litellm.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "litellm.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "litellm.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "litellm.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/infra/litellm/templates/_helpers.tpl b/infra/litellm/templates/_helpers.tpl new file mode 100644 index 0000000..a1eda28 --- /dev/null +++ b/infra/litellm/templates/_helpers.tpl @@ -0,0 +1,84 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "litellm.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "litellm.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "litellm.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "litellm.labels" -}} +helm.sh/chart: {{ include "litellm.chart" . }} +{{ include "litellm.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "litellm.selectorLabels" -}} +app.kubernetes.io/name: {{ include "litellm.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "litellm.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "litellm.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Get redis service name +*/}} +{{- define "litellm.redis.serviceName" -}} +{{- if and (eq .Values.redis.architecture "standalone") .Values.redis.sentinel.enabled -}} +{{- printf "%s-%s" .Release.Name (default "redis" .Values.redis.nameOverride | trunc 63 | trimSuffix "-") -}} +{{- else -}} +{{- printf "%s-%s-master" .Release.Name (default "redis" .Values.redis.nameOverride | trunc 63 | trimSuffix "-") -}} +{{- end -}} +{{- end -}} + +{{/* +Get redis service port +*/}} +{{- define "litellm.redis.port" -}} +{{- if .Values.redis.sentinel.enabled -}} +{{ .Values.redis.sentinel.service.ports.sentinel }} +{{- else -}} +{{ .Values.redis.master.service.ports.redis }} +{{- end -}} +{{- end -}} diff --git a/infra/litellm/templates/configmap-litellm.yaml b/infra/litellm/templates/configmap-litellm.yaml new file mode 100644 index 0000000..4598054 --- /dev/null +++ b/infra/litellm/templates/configmap-litellm.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "litellm.fullname" . }}-config +data: + config.yaml: | +{{ .Values.proxy_config | toYaml | indent 6 }} \ No newline at end of file diff --git a/infra/litellm/templates/deployment.yaml b/infra/litellm/templates/deployment.yaml new file mode 100644 index 0000000..697148a --- /dev/null +++ b/infra/litellm/templates/deployment.yaml @@ -0,0 +1,182 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "litellm.fullname" . }} + labels: + {{- include "litellm.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "litellm.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap-litellm.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "litellm.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "litellm.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ include "litellm.name" . }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "main-%s" .Chart.AppVersion) }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: HOST + value: "{{ .Values.listen | default "0.0.0.0" }}" + - name: PORT + value: {{ .Values.service.port | quote}} + {{- if .Values.db.deployStandalone }} + - name: DATABASE_USERNAME + valueFrom: + secretKeyRef: + name: {{ include "litellm.fullname" . }}-dbcredentials + key: username + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "litellm.fullname" . }}-dbcredentials + key: password + - name: DATABASE_HOST + value: {{ .Release.Name }}-postgresql + - name: DATABASE_NAME + value: litellm + {{- else if .Values.db.useExisting }} + - name: DATABASE_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.usernameKey }} + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.passwordKey }} + - name: DATABASE_HOST + value: {{ .Values.db.endpoint }} + - name: DATABASE_NAME + value: {{ .Values.db.database }} + - name: DATABASE_URL + value: {{ .Values.db.url | quote }} + {{- end }} + - name: PROXY_MASTER_KEY + valueFrom: + secretKeyRef: + name: {{ include "litellm.fullname" . }}-masterkey + key: masterkey + {{- if .Values.redis.enabled }} + - name: REDIS_HOST + value: {{ include "litellm.redis.serviceName" . }} + - name: REDIS_PORT + value: {{ include "litellm.redis.port" . | quote }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "redis.secretName" .Subcharts.redis }} + key: {{include "redis.secretPasswordKey" .Subcharts.redis }} + {{- end }} + {{- if .Values.envVars }} + {{- range $key, $val := .Values.envVars }} + - name: {{ $key }} + value: {{ $val | quote }} + {{- end }} + {{- end }} + envFrom: + {{- range .Values.environmentSecrets }} + - secretRef: + name: {{ . }} + {{- end }} + {{- range .Values.environmentConfigMaps }} + - configMapRef: + name: {{ . }} + {{- end }} + args: + - --config + - /etc/litellm/config.yaml + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + livenessProbe: + httpGet: + path: /health/liveliness + port: http + readinessProbe: + httpGet: + path: /health/readiness + port: http + # Give the container time to start up. Up to 5 minutes (10 * 30 seconds) + startupProbe: + httpGet: + path: /health/readiness + port: http + failureThreshold: 30 + periodSeconds: 10 + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: litellm-config + mountPath: /etc/litellm/ + {{ if .Values.securityContext.readOnlyRootFilesystem }} + - name: tmp + mountPath: /tmp + - name: cache + mountPath: /.cache + - name: npm + mountPath: /.npm + {{- end }} + {{- with .Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.extraContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{ if .Values.securityContext.readOnlyRootFilesystem }} + - name: tmp + emptyDir: + sizeLimit: 500Mi + - name: cache + emptyDir: + sizeLimit: 500Mi + - name: npm + emptyDir: + sizeLimit: 500Mi + {{- end }} + - name: litellm-config + configMap: + name: {{ include "litellm.fullname" . }}-config + items: + - key: "config.yaml" + path: "config.yaml" + {{- with .Values.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/infra/litellm/templates/hpa.yaml b/infra/litellm/templates/hpa.yaml new file mode 100644 index 0000000..71e199c --- /dev/null +++ b/infra/litellm/templates/hpa.yaml @@ -0,0 +1,32 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "litellm.fullname" . }} + labels: + {{- include "litellm.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "litellm.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/infra/litellm/templates/ingress.yaml b/infra/litellm/templates/ingress.yaml new file mode 100644 index 0000000..09e8d71 --- /dev/null +++ b/infra/litellm/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "litellm.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "litellm.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/infra/litellm/templates/migrations-job.yaml b/infra/litellm/templates/migrations-job.yaml new file mode 100644 index 0000000..e994c45 --- /dev/null +++ b/infra/litellm/templates/migrations-job.yaml @@ -0,0 +1,70 @@ +{{- if .Values.migrationJob.enabled }} +# This job runs the prisma migrations for the LiteLLM DB. +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "litellm.fullname" . }}-migrations + annotations: + argocd.argoproj.io/hook: PreSync + argocd.argoproj.io/hook-delete-policy: BeforeHookCreation # delete old migration on a new deploy in case the migration needs to make updates + checksum/config: {{ toYaml .Values | sha256sum }} +spec: + template: + metadata: + annotations: + {{- with .Values.migrationJob.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + containers: + - name: prisma-migrations + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "main-%s" .Chart.AppVersion) }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + command: ["python", "litellm/proxy/prisma_migration.py"] + workingDir: "/app" + env: + {{- if .Values.db.useExisting }} + - name: DATABASE_USERNAME + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.usernameKey }} + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.db.secret.name }} + key: {{ .Values.db.secret.passwordKey }} + - name: DATABASE_HOST + value: {{ .Values.db.endpoint }} + - name: DATABASE_NAME + value: {{ .Values.db.database }} + - name: DATABASE_URL + value: {{ .Values.db.url | quote }} + {{- else }} + - name: DATABASE_URL + value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }} + {{- end }} + - name: DISABLE_SCHEMA_UPDATE + value: "false" # always run the migration from the Helm PreSync hook, override the value set + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} + restartPolicy: OnFailure + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }} + backoffLimit: {{ .Values.migrationJob.backoffLimit }} +{{- end }} diff --git a/infra/litellm/templates/secret-dbcredentials.yaml b/infra/litellm/templates/secret-dbcredentials.yaml new file mode 100644 index 0000000..8851f58 --- /dev/null +++ b/infra/litellm/templates/secret-dbcredentials.yaml @@ -0,0 +1,12 @@ +{{- if .Values.db.deployStandalone -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "litellm.fullname" . }}-dbcredentials +data: + # Password for the "postgres" user + postgres-password: {{ ( index .Values.postgresql.auth "postgres-password") | default "litellm" | b64enc }} + username: {{ .Values.postgresql.auth.username | default "litellm" | b64enc }} + password: {{ .Values.postgresql.auth.password | default "litellm" | b64enc }} +type: Opaque +{{- end -}} \ No newline at end of file diff --git a/infra/litellm/templates/secret-masterkey.yaml b/infra/litellm/templates/secret-masterkey.yaml new file mode 100644 index 0000000..57b854c --- /dev/null +++ b/infra/litellm/templates/secret-masterkey.yaml @@ -0,0 +1,8 @@ +{{ $masterkey := (.Values.masterkey | default (randAlphaNum 17)) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "litellm.fullname" . }}-masterkey +data: + masterkey: {{ $masterkey | b64enc }} +type: Opaque \ No newline at end of file diff --git a/infra/litellm/templates/service.yaml b/infra/litellm/templates/service.yaml new file mode 100644 index 0000000..40e7f27 --- /dev/null +++ b/infra/litellm/templates/service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "litellm.fullname" . }} + labels: + {{- include "litellm.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "litellm.selectorLabels" . | nindent 4 }} diff --git a/infra/litellm/templates/serviceaccount.yaml b/infra/litellm/templates/serviceaccount.yaml new file mode 100644 index 0000000..7655470 --- /dev/null +++ b/infra/litellm/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "litellm.serviceAccountName" . }} + labels: + {{- include "litellm.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/infra/litellm/templates/tests/test-connection.yaml b/infra/litellm/templates/tests/test-connection.yaml new file mode 100644 index 0000000..86a8f66 --- /dev/null +++ b/infra/litellm/templates/tests/test-connection.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "litellm.fullname" . }}-test-connection" + labels: + {{- include "litellm.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['sh', '-c'] + args: + - | + # Wait for a bit to allow the service to be ready + sleep 10 + # Try multiple times with a delay between attempts + for i in $(seq 1 30); do + wget -T 5 "{{ include "litellm.fullname" . }}:{{ .Values.service.port }}/health/readiness" && exit 0 + echo "Attempt $i failed, waiting..." + sleep 2 + done + exit 1 + restartPolicy: Never \ No newline at end of file diff --git a/infra/litellm/templates/tests/test-env-vars.yaml b/infra/litellm/templates/tests/test-env-vars.yaml new file mode 100644 index 0000000..9f02775 --- /dev/null +++ b/infra/litellm/templates/tests/test-env-vars.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "litellm.fullname" . }}-env-test" + labels: + {{- include "litellm.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: test + image: busybox + command: ['sh', '-c'] + args: + - | + # Test DD_ENV + if [ "$DD_ENV" != "dev_helm" ]; then + echo "❌ Environment variable DD_ENV mismatch. Expected: dev_helm, Got: $DD_ENV" + exit 1 + fi + echo "✅ Environment variable DD_ENV matches expected value: $DD_ENV" + + # Test DD_SERVICE + if [ "$DD_SERVICE" != "litellm" ]; then + echo "❌ Environment variable DD_SERVICE mismatch. Expected: litellm, Got: $DD_SERVICE" + exit 1 + fi + echo "✅ Environment variable DD_SERVICE matches expected value: $DD_SERVICE" + + # Test USE_DDTRACE + if [ "$USE_DDTRACE" != "true" ]; then + echo "❌ Environment variable USE_DDTRACE mismatch. Expected: true, Got: $USE_DDTRACE" + exit 1 + fi + echo "✅ Environment variable USE_DDTRACE matches expected value: $USE_DDTRACE" + env: + - name: DD_ENV + value: {{ .Values.envVars.DD_ENV | quote }} + - name: DD_SERVICE + value: {{ .Values.envVars.DD_SERVICE | quote }} + - name: USE_DDTRACE + value: {{ .Values.envVars.USE_DDTRACE | quote }} + restartPolicy: Never \ No newline at end of file diff --git a/infra/litellm/values.yaml b/infra/litellm/values.yaml index b7ce066..2ea3221 100644 --- a/infra/litellm/values.yaml +++ b/infra/litellm/values.yaml @@ -1,197 +1,196 @@ -litellm: - # Default values for litellm. - # This is a YAML-formatted file. - # Declare variables to be passed into your templates. +# Default values for litellm. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. - replicaCount: 1 +replicaCount: 1 - image: - # Use "ghcr.io/berriai/litellm-database" for optimized image with database - repository: ghcr.io/berriai/litellm-database - pullPolicy: Always - # Overrides the image tag whose default is the chart appVersion. - # tag: "main-latest" - tag: "" +image: + # Use "ghcr.io/berriai/litellm-database" for optimized image with database + repository: ghcr.io/berriai/litellm-database + pullPolicy: Always + # Overrides the image tag whose default is the chart appVersion. + # tag: "main-latest" + tag: "" - imagePullSecrets: [] - nameOverride: "litellm" - fullnameOverride: "" +imagePullSecrets: [] +nameOverride: "litellm" +fullnameOverride: "" - serviceAccount: - # Specifies whether a service account should be created - create: false - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" +serviceAccount: + # Specifies whether a service account should be created + create: false + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" - podAnnotations: {} - podLabels: {} +podAnnotations: {} +podLabels: {} - # At the time of writing, the litellm docker image requires write access to the - # filesystem on startup so that prisma can install some dependencies. - podSecurityContext: {} - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: false - # runAsNonRoot: true - # runAsUser: 1000 +# At the time of writing, the litellm docker image requires write access to the +# filesystem on startup so that prisma can install some dependencies. +podSecurityContext: {} +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: false + # runAsNonRoot: true + # runAsUser: 1000 - # A list of Kubernetes Secret objects that will be exported to the LiteLLM proxy - # pod as environment variables. These secrets can then be referenced in the - # configuration file (or "litellm" ConfigMap) with `os.environ/` - environmentSecrets: [] - # - litellm-env-secret +# A list of Kubernetes Secret objects that will be exported to the LiteLLM proxy +# pod as environment variables. These secrets can then be referenced in the +# configuration file (or "litellm" ConfigMap) with `os.environ/` +environmentSecrets: [] + # - litellm-env-secret - # A list of Kubernetes ConfigMap objects that will be exported to the LiteLLM proxy - # pod as environment variables. The ConfigMap kv-pairs can then be referenced in the - # configuration file (or "litellm" ConfigMap) with `os.environ/` - environmentConfigMaps: [] - # - litellm-env-configmap +# A list of Kubernetes ConfigMap objects that will be exported to the LiteLLM proxy +# pod as environment variables. The ConfigMap kv-pairs can then be referenced in the +# configuration file (or "litellm" ConfigMap) with `os.environ/` +environmentConfigMaps: [] + # - litellm-env-configmap - service: - type: ClusterIP - port: 4000 +service: + type: ClusterIP + port: 4000 - ingress: - enabled: false - className: "nginx" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: api.example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local +ingress: + enabled: false + className: "nginx" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: api.example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local - # masterkey: changeit +# masterkey: changeit - # The elements within proxy_config are rendered as config.yaml for the proxy - # Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml - # Reference: https://docs.litellm.ai/docs/proxy/configs - proxy_config: - model_list: - # At least one model must exist for the proxy to start. - - model_name: gpt-3.5-turbo - litellm_params: - model: gpt-3.5-turbo - api_key: eXaMpLeOnLy - - model_name: fake-openai-endpoint - litellm_params: - model: openai/fake - api_key: fake-key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - general_settings: - master_key: os.environ/PROXY_MASTER_KEY +# The elements within proxy_config are rendered as config.yaml for the proxy +# Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml +# Reference: https://docs.litellm.ai/docs/proxy/configs +proxy_config: + model_list: + # At least one model must exist for the proxy to start. + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo + api_key: eXaMpLeOnLy + - model_name: fake-openai-endpoint + litellm_params: + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + general_settings: + master_key: os.environ/PROXY_MASTER_KEY - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi - autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 - # Additional volumes on the output Deployment definition. - volumes: [] - # - name: foo - # secret: - # secretName: mysecret - # optional: false +# Additional volumes on the output Deployment definition. +volumes: [] +# - name: foo +# secret: +# secretName: mysecret +# optional: false - # Additional volumeMounts on the output Deployment definition. - volumeMounts: [] - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true +# Additional volumeMounts on the output Deployment definition. +volumeMounts: [] +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true - nodeSelector: {} +nodeSelector: {} - tolerations: [] +tolerations: [] - affinity: {} +affinity: {} - db: - # Use an existing postgres server/cluster - useExisting: false +db: + # Use an existing postgres server/cluster + useExisting: false - # How to connect to the existing postgres server/cluster - endpoint: localhost + # How to connect to the existing postgres server/cluster + endpoint: localhost + database: litellm + url: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME) + secret: + name: postgres + usernameKey: username + passwordKey: password + + # Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster. + # The Stackgres Operator must already be installed within the target + # Kubernetes cluster. + # TODO: Stackgres deployment currently unsupported + useStackgresOperator: false + + # Use the Postgres Helm chart to create a single node, stand alone postgres + # instance. See the "postgresql" top level key for additional configuration. + deployStandalone: true + +# Settings for Bitnami postgresql chart (if db.deployStandalone is true, ignored +# otherwise) +postgresql: + architecture: standalone + auth: + username: litellm database: litellm - url: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME) - secret: - name: postgres - usernameKey: username - passwordKey: password - # Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster. - # The Stackgres Operator must already be installed within the target - # Kubernetes cluster. - # TODO: Stackgres deployment currently unsupported - useStackgresOperator: false + # You should override these on the helm command line with + # `--set postgresql.auth.postgres-password=,postgresql.auth.password=` + password: NoTaGrEaTpAsSwOrD + postgres-password: NoTaGrEaTpAsSwOrD - # Use the Postgres Helm chart to create a single node, stand alone postgres - # instance. See the "postgresql" top level key for additional configuration. - deployStandalone: true + # A secret is created by this chart (litellm-helm) with the credentials that + # the new Postgres instance should use. + # existingSecret: "" + # secretKeys: + # userPasswordKey: password - # Settings for Bitnami postgresql chart (if db.deployStandalone is true, ignored - # otherwise) - postgresql: - architecture: standalone - auth: - username: litellm - database: litellm +# requires cache: true in config file +# either enable this or pass a secret for REDIS_HOST, REDIS_PORT, REDIS_PASSWORD or REDIS_URL +# with cache: true to use existing redis instance +redis: + enabled: false + architecture: standalone - # You should override these on the helm command line with - # `--set postgresql.auth.postgres-password=,postgresql.auth.password=` - password: NoTaGrEaTpAsSwOrD - postgres-password: NoTaGrEaTpAsSwOrD +# Prisma migration job settings +migrationJob: + enabled: true # Enable or disable the schema migration Job + retries: 3 # Number of retries for the Job in case of failure + backoffLimit: 4 # Backoff limit for Job restarts + disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0. + annotations: {} + ttlSecondsAfterFinished: 120 - # A secret is created by this chart (litellm-helm) with the credentials that - # the new Postgres instance should use. - # existingSecret: "" - # secretKeys: - # userPasswordKey: password - - # requires cache: true in config file - # either enable this or pass a secret for REDIS_HOST, REDIS_PORT, REDIS_PASSWORD or REDIS_URL - # with cache: true to use existing redis instance - redis: - enabled: false - architecture: standalone - - # Prisma migration job settings - migrationJob: - enabled: true # Enable or disable the schema migration Job - retries: 3 # Number of retries for the Job in case of failure - backoffLimit: 4 # Backoff limit for Job restarts - disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0. - annotations: {} - ttlSecondsAfterFinished: 120 - - # Additional environment variables to be added to the deployment - envVars: { - # USE_DDTRACE: "true" - } +# Additional environment variables to be added to the deployment +envVars: { + # USE_DDTRACE: "true" +}