Feature/add django chart on helm depedencies

.dockerignore

@@ -129,3 +129,8 @@ dmypy.json
 
 # editors
 .idea/
+
+# helm
+./helm/charts/
+./helm/.helm-charts/
+./helm/values-local.yaml

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-extend-ignore = C901, W504
+extend-ignore = C901, W504, E701
 max-line-length = 125
 # NOTE: Update in .pre-commit-config.yaml as well
 extend-exclude = .git,__pycache__,old,build,dist,*/migrations/*.py,legacy/,.venv

.github/workflows/ci.yml

@@ -53,8 +53,8 @@ jobs:
 
           # XXX: Check if there is a slash in the BRANCH_NAME eg: project/add-docker
           if [[ "$BRANCH_NAME" == *"/"* ]]; then
-              # XXX: Change the docker image package to -alpha
-              IMAGE_NAME="$IMAGE_NAME-alpha"
+              # XXX: Change the docker image package to -dev
+              IMAGE_NAME="$IMAGE_NAME-dev"
               TAG="$(echo "$BRANCH_NAME" | sed 's|/|-|g').$(echo $GITHUB_SHA | head -c7)"
           else
               TAG="$BRANCH_NAME.$(echo $GITHUB_SHA | head -c7)"
@@ -124,8 +124,8 @@ jobs:
 
       - name: 🐳 Helm dependency
         run: |
-          yq --indent 0 '.dependencies | map(["helm", "repo", "add", .name, .repository] | join(" ")) | .[]' ./helm/Chart.lock  | sh --
-          helm dependency build ./helm
+          yq --indent 0 '.dependencies | map(select(.repository | test("^oci:") | not)) | map(["helm", "repo", "add", .name, .repository] | join(" ")) | .[]' ./helm/Chart.lock | sh --
+          helm dependency build ./helm/
 
       - name: 🐳 Helm lint
         run: helm lint ./helm --values ./helm/values-test.yaml

.github/workflows/helm-publish.yml

@@ -3,7 +3,7 @@ name: Builds and pushes Docker Images and Helm charts to Github Registry
 on:
   workflow_dispatch:
 
-      # XXX: To add tags: Update the -alpha logic
+      # XXX: To add tags: Update the -dev logic
 
 permissions:
   packages: write
@@ -36,8 +36,8 @@ jobs:
 
     - name: 🐳 Helm dependency
       run: |
-        yq --indent 0 '.dependencies | map(["helm", "repo", "add", .name, .repository] | join(" ")) | .[]' ./helm/Chart.lock  | sh --
-        helm dependency build ./helm
+        yq --indent 0 '.dependencies | map(select(.repository | test("^oci:") | not)) | map(["helm", "repo", "add", .name, .repository] | join(" ")) | .[]' ./helm/Chart.lock | sh --
+        helm dependency build ./helm/
 
     - name: Tag docker image in Helm Chart values.yaml
       env:
@@ -50,29 +50,38 @@ jobs:
 
     - name: Package Helm Chart
       id: set-variables
+      env:
+        IMAGE_TAG: ${{ needs.ci.outputs.docker_image_tag }}
       run: |
         # XXX: Check if there is a slash in the BRANCH_NAME eg: project/add-docker
         if [[ "$GITHUB_REF_NAME" == *"/"* ]]; then
-            # XXX: Change the helm chart to <chart-name>-alpha
-            sed -i 's/^name: \(.*\)/name: \1-alpha/' helm/Chart.yaml
+            # XXX: Change the helm chart to <chart-dev-name>
+            sed -i 's/^\(name:.*\)-helm$/\1-dev-helm/' ./helm/Chart.yaml
         fi
 
         SHA_SHORT=$(git rev-parse --short HEAD)
         sed -i "s/SET-BY-CICD/$SHA_SHORT/g" helm/Chart.yaml
-        helm package ./helm -d .helm-charts
+        helm package ./helm/ -d ./helm/.helm-charts
 
     - name: Push Helm Chart
       env:
         IMAGE: ${{ needs.ci.outputs.docker_image }}
-        OCI_REPO: oci://ghcr.io/${{ github.repository }}
+        OCI_REPO: oci://ghcr.io/${{ github.repository_owner }}
       run: |
-        PACKAGE_FILE=$(ls .helm-charts/*.tgz | head -n 1)
+        OCI_REPO=$(echo $OCI_REPO | tr '[:upper:]' '[:lower:]')
+        PACKAGE_FILE=$(ls ./helm/.helm-charts/*.tgz | head -n 1)
         echo "# Helm Chart" >> $GITHUB_STEP_SUMMARY
         echo "" >> $GITHUB_STEP_SUMMARY
-        echo "Tagged Image: **$IMAGE**" >> $GITHUB_STEP_SUMMARY
-        echo "" >> $GITHUB_STEP_SUMMARY
-        echo "Helm push output" >> $GITHUB_STEP_SUMMARY
-        echo "" >> $GITHUB_STEP_SUMMARY
-        echo '```bash' >> $GITHUB_STEP_SUMMARY
-        helm push "$PACKAGE_FILE" $OCI_REPO >> $GITHUB_STEP_SUMMARY
+        echo '```yaml' >> $GITHUB_STEP_SUMMARY
+        helm push "$PACKAGE_FILE" $OCI_REPO 2>> $GITHUB_STEP_SUMMARY
         echo '```' >> $GITHUB_STEP_SUMMARY
+        echo "> [!Important]" >> $GITHUB_STEP_SUMMARY
+        echo "> Helm Repo URL: **$OCI_REPO**" >> $GITHUB_STEP_SUMMARY
+        echo "> Helm Chart: **$(helm show chart "$PACKAGE_FILE" | grep '^name:' | awk '{print $2}')**" >> $GITHUB_STEP_SUMMARY
+        echo "> Helm Target Revision: **$(helm show chart "$PACKAGE_FILE" | grep '^version:' | awk '{print $2}')**" >> $GITHUB_STEP_SUMMARY
+        echo "> Docker image: **$IMAGE**" >> $GITHUB_STEP_SUMMARY
+
+        # Add annotations as well
+        echo "::notice::Helm Repo URL: **$OCI_REPO**"
+        echo "::notice::Helm Chart: **$(helm show chart "$PACKAGE_FILE" | grep '^name:' | awk '{print $2}')**"
+        echo "::notice::Helm Target Revision: **$(helm show chart "$PACKAGE_FILE" | grep '^version:' | awk '{print $2}')**"

common/management/commands/wait_for_resources.py

@@ -0,0 +1,137 @@
+import signal
+import time
+from urllib.parse import urljoin
+
+import requests
+from django.conf import settings
+from django.core.cache import cache
+from django.core.management.base import BaseCommand
+from django.db import connections
+from django.db.utils import OperationalError
+from redis.exceptions import ConnectionError as RedisConnectionError
+
+
+class TimeoutException(Exception): ...
+
+
+class RetryHelper:
+    def __init__(self, base_wait_seconds: int = 2, wait_max_seconds: int = 60):
+        self.base_wait_seconds = base_wait_seconds
+        self.wait_max_seconds = wait_max_seconds
+        self.attempt = 1
+        self.next_wait = base_wait_seconds
+        self.start_time = time.time()
+
+    def next_wait_seconds(self) -> int:
+        return self.next_wait
+
+    def wait(self) -> None:
+        time.sleep(self.next_wait)
+        self.attempt += 1
+        if self.next_wait < self.wait_max_seconds:
+            self.next_wait = self.base_wait_seconds**self.attempt
+        else:
+            self.next_wait = self.wait_max_seconds
+
+    def total_time(self) -> float:
+        return time.time() - self.start_time
+
+    def try_again_message(self, prefix: str) -> str:
+        return f"{prefix}, Attempt: {self.attempt}, try again after {self.next_wait_seconds()} seconds..."
+
+
+def timeout_handler(*_):
+    raise Exception("The command timed out.")
+
+
+class Command(BaseCommand):
+    help = "Wait for resources our application depends on"
+
+    def wait_for_db(self):
+        self.stdout.write("Waiting for DB...")
+        db_conn = None
+        retry_helper = RetryHelper()
+        while True:
+            try:
+                db_conn = connections["default"]
+                db_conn.ensure_connection()
+                break
+            except OperationalError:
+                ...
+            # Try again
+            self.stdout.write(self.style.WARNING(retry_helper.try_again_message("DB not available")))
+            retry_helper.wait()
+
+        self.stdout.write(self.style.SUCCESS(f"DB is available after {retry_helper.total_time()} seconds"))
+
+    def wait_for_redis(self):
+        self.stdout.write("Waiting for Redis...")
+        redis_conn = None
+        retry_helper = RetryHelper()
+        while True:
+            try:
+                cache.set("wait-for-it-ping", "pong", timeout=1)  # Set a key to check Redis availability
+                redis_conn = cache.get("wait-for-it-ping")  # Try to get the value back from Redis
+                if redis_conn != "pong":
+                    raise TypeError
+                break
+            except (RedisConnectionError, TypeError):
+                ...
+            # Try again
+            self.stdout.write(self.style.WARNING(retry_helper.try_again_message("Redis not available")))
+            retry_helper.wait()
+
+        self.stdout.write(self.style.SUCCESS(f"Redis is available after {retry_helper.total_time()} seconds"))
+
+    def wait_for_minio(self):
+        self.stdout.write("Waiting for Minio...")
+        endpoint_url = getattr(settings, "AWS_S3_ENDPOINT_URL", None)
+        if endpoint_url is None:
+            self.stdout.write(self.style.WARNING("No endpoint_url is provided. Skipping wait"))
+            return
+
+        retry_helper = RetryHelper()
+        while True:
+            try:
+                response = requests.get(urljoin(endpoint_url, "/minio/health/live"), timeout=5)
+                if response.status_code == 200:
+                    break
+            except requests.exceptions.RequestException:
+                ...
+            # Try again
+            self.stdout.write(self.style.WARNING(retry_helper.try_again_message("Minio not available")))
+            retry_helper.wait()
+
+        self.stdout.write(self.style.SUCCESS(f"Minio is available after {retry_helper.total_time()} seconds"))
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--timeout",
+            type=int,
+            default=600,
+            help="The maximum time (in seconds) the command is allowed to run before timing out. Default is 10 min.",
+        )
+        parser.add_argument("--db", action="store_true", help="Wait for DB to be available")
+        parser.add_argument("--redis", action="store_true", help="Wait for Redis to be available")
+        parser.add_argument("--minio", action="store_true", help="Wait for MinIO (S3) storage to be available")
+        parser.add_argument("--all", action="store_true", help="Wait for all to be available")
+
+    def handle(self, **kwargs):
+        timeout = kwargs["timeout"]
+        _all = kwargs["all"]
+
+        signal.signal(signal.SIGALRM, timeout_handler)
+        signal.alarm(timeout)
+
+        try:
+            if _all or kwargs["db"]:
+                self.wait_for_db()
+            if _all or kwargs["minio"]:
+                self.wait_for_minio()
+            if _all or kwargs["redis"]:
+                self.wait_for_redis()
+        except TimeoutException:
+            ...
+        finally:
+            # Disable the alarm (cleanup)
+            signal.alarm(0)

helm/.helmignore

@@ -0,0 +1 @@
+values-local.yaml

helm/Chart.lock

@@ -1,12 +1,9 @@
 dependencies:
-- name: redis
-  repository: https://charts.bitnami.com/bitnami
-  version: 20.2.1
-- name: postgresql
-  repository: https://charts.bitnami.com/bitnami
-  version: 16.1.2
 - name: qdrant
   repository: https://qdrant.github.io/qdrant-helm
   version: 1.12.2
-digest: sha256:3b94766cdd43351699a96ec2bebc27656e34de13c49c7be279db63f750cde6ae
-generated: "2024-11-10T23:44:51.273569466+05:45"
+- name: django-app
+  repository: oci://ghcr.io/toggle-corp/charts
+  version: 0.1.1
+digest: sha256:dced24a0e6978f629a33eaf57fbcba99d1b902160b6bb3f626bb8da2c272b5ca
+generated: "2025-05-06T15:56:53.669786475+05:45"

helm/Chart.yaml

@@ -6,22 +6,16 @@ version: 0.0.1-SET-BY-CICD
 sources:
   - https://github.com/toggle-corp/ai-chatbot-backend
 
-
 dependencies:
-  - name: redis
-    alias: redis
-    version: "20.2.1"
-    repository: "https://charts.bitnami.com/bitnami"
-    condition: redis.enabled
-
-  - name: postgresql
-    alias: postgresql
-    version: "16.1.2"
-    repository: "https://charts.bitnami.com/bitnami"
-    condition: postgresql.enabled
-
   - name: qdrant
     alias: qdrant
     version: "1.12.2"
     repository: "https://qdrant.github.io/qdrant-helm"
     condition: qdrant.enabled
+
+  - name: django-app
+    alias: app
+    version: 0.1.1
+    repository: oci://ghcr.io/toggle-corp/charts\
+
+