feature(pdf2md): added pdf2md helm chart

cdxker · cdxker · commit 4a446bf8d615 · 2025-03-17T03:25:22.000-07:00
diff --git a/charts/pdf2md/Chart.lock b/charts/pdf2md/Chart.lock
@@ -0,0 +1,9 @@
+dependencies:
+- name: redis
+  repository: https://charts.bitnami.com/bitnami
+  version: 20.10.0
+- name: clickhouse
+  repository: ""
+  version: 0.0.0
+digest: sha256:ab1d96b6b226666e40dbc914a2c9c6fd60ad3bce71b0ba97085f3f9fa03f9090
+generated: "2025-03-16T23:53:51.662732432-07:00"
diff --git a/charts/pdf2md/Chart.yaml b/charts/pdf2md/Chart.yaml
@@ -0,0 +1,14 @@
+apiVersion: v2
+name: pdf2md
+description: Helm chart for pdf2md pdf2md.trieve.ai
+type: application
+version: 0.0.1
+appVersion: "sha-b6286ae"
+dependencies:
+  - name: redis
+    repository: https://charts.bitnami.com/bitnami
+    version: 20.10.0
+    condition: redis.enabled
+  - name: clickhouse
+    version: 0.0.0
+    condition: clickhouse.enabled
diff --git a/charts/pdf2md/charts/clickhouse/.helmignore b/charts/pdf2md/charts/clickhouse/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/charts/pdf2md/charts/clickhouse/Chart.lock b/charts/pdf2md/charts/clickhouse/Chart.lock
@@ -0,0 +1,6 @@
+dependencies:
+- name: altinity-clickhouse-operator
+  repository: https://docs.altinity.com/clickhouse-operator/
+  version: 0.24.4
+digest: sha256:fee75e1786d42b6b0893ecc4a75bca13b539c6f71fabca82fa1690d0f1aa661a
+generated: "2025-03-02T00:44:43.875908354-08:00"
diff --git a/charts/pdf2md/charts/clickhouse/Chart.yaml b/charts/pdf2md/charts/clickhouse/Chart.yaml
@@ -0,0 +1,8 @@
+name: clickhouse
+apiVersion: v2
+version: 0.0.0
+dependencies:
+  - name: altinity-clickhouse-operator
+    version: 0.24.4
+    repository: https://docs.altinity.com/clickhouse-operator/
+    condition: installCrds
diff --git a/charts/pdf2md/charts/clickhouse/charts/altinity-clickhouse-operator-0.24.4.tgz b/charts/pdf2md/charts/clickhouse/charts/altinity-clickhouse-operator-0.24.4.tgz
diff --git a/charts/pdf2md/charts/clickhouse/templates/clickhouse-installation.yaml b/charts/pdf2md/charts/clickhouse/templates/clickhouse-installation.yaml
@@ -0,0 +1,10 @@
+apiVersion: "clickhouse.altinity.com/v1"
+kind: "ClickHouseInstallation"
+metadata:
+  name: {{ $.Release.Name }}-trieve
+  labels:
+    app.kubernetes.io/name: {{ $.Release.Name }}-trieve
+    app.kubernetes.io/instance: {{ .Release.Name }}
+spec: 
+{{ toYaml .Values.clickhouseSpec | indent 2 }}
+
diff --git a/charts/pdf2md/charts/clickhouse/values.yaml b/charts/pdf2md/charts/clickhouse/values.yaml
@@ -0,0 +1,11 @@
+
+installCrds: true
+
+clickhouseSpec: 
+  ## A ClickhouseInstallation template spec
+  configuration:
+    users:
+      # printf 'test_password' | sha256sum
+      test_user/password_sha256_hex: 10a6e6cc8311a3e2bcc09bf6c199adecd5dd59408c343e926b129c4914f3cb01
+    clusters:
+      - name: "simple"
diff --git a/charts/pdf2md/charts/redis-20.10.0.tgz b/charts/pdf2md/charts/redis-20.10.0.tgz
diff --git a/charts/pdf2md/templates/_helpers.tpl b/charts/pdf2md/templates/_helpers.tpl
@@ -0,0 +1,14 @@
+{{- define "pdf2md.redisUrl" -}}
+{{- if .Values.redis.enabled -}}
+{{- $svcName := printf "%s-redis-master" .Release.Name -}}
+{{- $redisSecretName := printf "%s-redis" .Release.Name -}}
+{{- $redisSecretKey := "redis-password" -}}
+{{- $redisUri := printf "redis://%s@%s" .Release.Name -}}
+{{- with (lookup "v1" "Secret" .Release.Namespace $redisSecretName) -}}
+{{- $redisPassword := index .data $redisSecretKey | b64dec -}}
+redis://:{{ $redisPassword }}@{{ $svcName }}:6379
+{{- end -}}
+{{- else -}}
+{{ .Values.redis.connection }}
+{{- end -}}
+{{- end -}}
diff --git a/charts/pdf2md/templates/configmap.yaml b/charts/pdf2md/templates/configmap.yaml
@@ -0,0 +1,33 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: pdf2md-config
+  labels:
+    app.kubernetes.io/instance: {{ $.Release.Name }}
+data:
+  REDIS_URL: {{ include "pdf2md.redisUrl" . | indent 2 }}
+
+  # Clickhouse
+  CLICKHOUSE_URL: {{ $.Values.clickhouse.connection.clickhouseUrl }}
+  CLICKHOUSE_DB: {{ $.Values.clickhouse.connection.clickhouseDB }}
+  CLICKHOUSE_USER: {{ $.Values.clickhouse.connection.clickhouseUser }}
+  CLICKHOUSE_PASSWORD: {{ $.Values.clickhouse.connection.clickhousePassword }}
+
+  # S3
+  S3_ENDPOINT: {{ $.Values.s3.endpoint }}
+  S3_ACCESS_KEY: {{ $.Values.s3.accessKey }}
+  S3_SECRET_KEY: {{ $.Values.s3.secretKey }}
+  S3_BUCKET: {{ $.Values.s3.bucket }}
+  AWS_REGION: {{ $.Values.s3.region }}
+
+  # PDF2MD conversion worker services
+  LLM_BASE_URL: {{ $.Values.config.llm.baseUrl }}
+  LLM_API_KEY: {{ $.Values.config.llm.apiKey }}
+  LLM_MODEL: {{ $.Values.config.llm.model }}
+
+  # PDF2MD HTTP API server
+  API_KEY: {{ $.Values.config.pdf2md.apiKey }}
+
+  CHUNKR_API_URL: {{ $.Values.config.chunkr.baseUrl }}
+  CHUNKR_API_KEY: {{ $.Values.config.chunkr.apiKey }}
+
diff --git a/charts/pdf2md/templates/pdf2md-deployments.yaml b/charts/pdf2md/templates/pdf2md-deployments.yaml
@@ -0,0 +1,46 @@
+{{- range $.Values.containers }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .name }}
+  labels:
+    app.kubernetes.io/name: {{ .name }}
+    app.kubernetes.io/instance: {{ $.Release.Name }}
+  annotations:
+    checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") $ | sha256sum }}
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ .name }}
+      app.kubernetes.io/instance: {{ $.Release.Name }}
+  template:
+    metadata:
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") $ | sha256sum }}
+      labels:
+        app.kubernetes.io/name: {{ .name }}
+        app.kubernetes.io/instance: {{ $.Release.Name }}
+    spec:
+      {{- if .imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml .imagePullSecrets | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: {{ .name }}
+          image: {{ printf "%s/%s:%s" (default "trieve" .image.registry) .image.repository (default $.Chart.AppVersion .image.tag) }}
+          {{- if .image.pullPolicy }}
+          imagePullPolicy: {{ .image.pullPolicy }}
+          {{- end }}
+          {{- if .ports }}
+          ports:
+            {{- toYaml .ports | nindent 10 }}
+          {{- end }}
+          {{- if .resources }}
+          resources:
+            {{- toYaml .resources | nindent 12 }}
+          {{- end }}
+          envFrom:
+            - configMapRef:
+                name: pdf2md-config
+---
+{{- end }}
diff --git a/charts/pdf2md/templates/pdf2md-service.yaml b/charts/pdf2md/templates/pdf2md-service.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: pdf2md-service
+  labels:
+    app.kubernetes.io/name: pdf2md
+    app.kubernetes.io/instance: {{ .Release.Name }}
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: pdf2md-server
+    app.kubernetes.io/instance: {{ .Release.Name }}
+  ports:
+    - name: http
+      port: 8081
+      targetPort: 8081
+      protocol: TCP
diff --git a/charts/pdf2md/values.yaml b/charts/pdf2md/values.yaml
@@ -0,0 +1,139 @@
+redis:
+  enabled: true
+  # connection: redis://:password@localhost:6379/0
+
+  metrics:
+    enabled: false
+    serviceMonitor:
+      enabled: false
+  auth:
+    password: "password"
+  master:
+    disableCommands: []
+    persistence:
+      enabled: false
+    pdb:
+      create: false
+    resources:
+      limits:
+        memory: 5Gi
+  replica:
+    replicaCount: 0
+
+clickhouse:
+  enabled: true
+  installCrds: true
+
+  connection:
+    # TODO modify password
+    clickhouseDB: default
+    clickhouseUser: default
+    clickhousePassword: cywtKFfCdRYq7H
+    # May change as release name changes
+    clickhouseUrl: http://clickhouse-pdf2md-trieve:8123
+
+  clickhouseSpec:
+    configuration:
+      users:
+        default/password: cywtKFfCdRYq7H
+        # to allow access outside from kubernetes
+        default/networks/ip:
+          - 0.0.0.0/0
+      clusters:
+        - name: "default"
+          layout:
+            shardsCount: 1
+            replicasCount: 1
+          templates:
+            podTemplate: clickhouse-stable
+            volumeClaimTemplate: storage-vc-template
+    templates:
+      podTemplates:
+        - name: clickhouse-stable
+          spec:
+            containers:
+            - name: clickhouse
+              image: clickhouse/clickhouse-server:24.12.6
+      volumeClaimTemplates:
+        - name: storage-vc-template
+          spec:
+            accessModes:
+              - ReadWriteOnce
+            resources:
+              requests:
+                storage: 20Gi
+
+s3:
+  endpoint: https://bucket-s3bucket.s3.amazonaws.com
+  accessKey: NXVN****************
+  secretKey: czc3L***********************************
+  bucket: pdf2md-s3bucket
+  region: us-east-2
+
+config:
+
+  llm:
+    # We recommend openrouter because they accept
+    # a larger payload size. 
+    baseUrl: https://openrouter.ai/api/v1
+    apiKey: sk-**********************************************************************
+    model: gpt-*******
+
+  chunkr:
+    baseUrl: https://api.chunkr.ai
+    apiKey: lu_*********************************************
+
+  pdf2md:
+    # PDF2MD has a single ADMIN API apiKey
+    # Change as needed
+    apiKey: CQS2ZQUGG43CNqn1CVfreire
+
+containers:
+  - name: pdf2md-server
+    
+    image:
+      repository: pdf2md-server
+      # registry: trieve
+      # If not set, default will be Chart.appVersion
+      # tag: sha-5243efe
+      pullPolicy: Always
+
+    # imagePullSecrets:
+    #   - name: trieve-registry
+    #
+    ports:
+      - name: http
+        containerPort: 8081
+
+    resources:
+      requests:
+        memory: "300Mi"
+      #   cpu: "100m"
+      # limits:
+      #   memory: "256Mi"
+      #   cpu: "200m"
+
+  - name: chunk-worker
+    image:
+      repository: chunk-worker
+      # registry: trieve
+      # If not set, default will be Chart.appVersion
+      # tag: sha-5243efe
+      pullPolicy: Always
+
+    # imagePullSecrets:
+    #   - name: trieve-registry
+    #
+    
+  - name: supervisor-worker
+    image:
+      repository: supervisor-worker
+      # registry: trieve
+      # If not set, default will be Chart.appVersion
+      # tag: sha-5243efe
+      pullPolicy: Always
+
+    # imagePullSecrets:
+      # o
+    #   - name: trieve-registry
+    #
diff --git a/charts/trieve/Chart.yaml b/charts/trieve/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: trieve
 description: Helm chart expressing Trieve deployment without any dependencies
 type: application
-version: 0.2.3
+version: 0.2.4
 appVersion: "0.12.0"
 dependencies:
   - name: qdrant
diff --git a/charts/trieve/templates/settings/_helpers.tpl b/charts/trieve/templates/settings/_helpers.tpl
@@ -6,7 +6,7 @@
 {{- $redisUri := printf "redis://%s@%s" .Release.Name -}}
 {{- with (lookup "v1" "Secret" .Release.Namespace $redisSecretName) -}}
 {{- $redisPassword := index .data $redisSecretKey | b64dec -}}
-redis://{{ $redisPassword }}@{{ $svcName }}
+redis://:{{ $redisPassword }}@{{ $svcName }}:6379
 {{- end -}}
 {{- else -}}
 {{ .Values.config.redis.uri }}
