Add storage provisioning diagnostics for CI failures

Author: sfc-gh-jmckulka

.github/actions/k3d/monitor-resources.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Monitor k3d cluster resources during tests
+# Run this in the background to capture resource usage over time
+
+set -euo pipefail
+
+INTERVAL="${1:-30}"  # Check every N seconds
+DURATION="${2:-300}" # Run for N seconds (5 minutes)
+OUTPUT="${3:-resource-monitor.log}"
+
+echo "Starting resource monitoring (interval=${INTERVAL}s, duration=${DURATION}s)" > "$OUTPUT"
+echo "================================================================" >> "$OUTPUT"
+
+END_TIME=$(($(date +%s) + DURATION))
+
+while [ $(date +%s) -lt $END_TIME ]; do
+    TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
+    
+    echo "" >> "$OUTPUT"
+    echo "=== $TIMESTAMP ===" >> "$OUTPUT"
+    
+    # Docker disk usage
+    echo "--- Docker Disk Usage ---" >> "$OUTPUT"
+    docker system df >> "$OUTPUT" 2>&1 || true
+    
+    # k3d node disk usage
+    echo "--- k3d Node Disk Usage ---" >> "$OUTPUT"
+    docker exec k3d-k3s-default-server-0 df -h 2>> "$OUTPUT" | \
+        grep -E '(Filesystem|/$|/var/lib|overlay)' >> "$OUTPUT" || true
+    
+    # PVC counts
+    echo "--- PVC Status ---" >> "$OUTPUT"
+    kubectl get pvc --all-namespaces -o json 2>/dev/null | \
+        jq -r '.items | group_by(.status.phase) | map({phase: .[0].status.phase, count: length}) | .[]' >> "$OUTPUT" 2>&1 || true
+    
+    # Local path provisioner status
+    echo "--- Provisioner Pods ---" >> "$OUTPUT"
+    kubectl get pods -n kube-system -l app=local-path-provisioner -o wide >> "$OUTPUT" 2>&1 || true
+    
+    sleep "$INTERVAL"
+done
+
+echo "Resource monitoring complete" >> "$OUTPUT"

.github/workflows/test.yaml

@@ -135,12 +135,48 @@ jobs:
             --env 'PGO_FEATURE_GATES=TablespaceVolumes=true,OpenTelemetryLogs=true,OpenTelemetryMetrics=true' \
             --name 'postgres-operator' localhost/postgres-operator
 
+      - name: Check initial environment health
+        run: |
+          echo "::group::Docker system info"
+          docker system df
+          docker info | grep -E 'Storage Driver|Data Space'
+          echo "::endgroup::"
+          echo "::group::k3d node disk space"
+          docker exec k3d-k3s-default-server-0 df -h
+          echo "::endgroup::"
+          echo "::group::Storage provisioner status"
+          kubectl get storageclass -o wide
+          kubectl get pods -n kube-system -l app=local-path-provisioner -o wide
+          echo "::endgroup::"
+
+      - name: Start resource monitoring
+        run: |
+          nohup .github/actions/k3d/monitor-resources.sh 30 600 chainsaw-resource-monitor.log &
+          echo $! > monitor.pid
+
       - run: |
           make check-chainsaw && exit
           failed=$?
           echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::'
           exit $failed
 
+      - name: Stop resource monitoring
+        if: always()
+        run: |
+          if [ -f monitor.pid ]; then
+            kill $(cat monitor.pid) 2>/dev/null || true
+            rm monitor.pid
+          fi
+
+      - name: Upload resource monitoring logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: "chainsaw-resource-monitor-k8s=${{ matrix.kubernetes }}"
+          path: chainsaw-resource-monitor.log
+          retention-days: 7
+          if-no-files-found: ignore
+
       - run: make generate-kuttl
         env:
           KUTTL_PG_UPGRADE_FROM_VERSION: '16'

testing/chainsaw/e2e/pgbackrest-restore/templates/clone-cluster.yaml

@@ -34,6 +34,19 @@ spec:
                     volume:
                       volumeClaimSpec: ($volume)
 
+    - description: Wait for PVCs to be created
+      sleep:
+        duration: 5s
+
+    - description: Check PVC provisioning status early
+      script:
+        content: |
+          echo "=== PVCs for ($name) ==="
+          kubectl get pvc -l postgres-operator.crunchydata.com/cluster=($name) -o wide || true
+          echo ""
+          echo "=== All PVC Events ==="
+          kubectl get events --field-selector involvedObject.kind=PersistentVolumeClaim -o wide | tail -20 || true
+
     -
       description: >
         Wait for the cluster to come online
@@ -89,3 +102,36 @@ spec:
         apiVersion: v1
         kind: PersistentVolumeClaim
         selector: (join('', ['postgres-operator.crunchydata.com/cluster=', $name]))
+
+    - description: Get all PVC events to diagnose provisioning failures
+      script:
+        content: kubectl get events --all-namespaces --field-selector involvedObject.kind=PersistentVolumeClaim -o wide
+
+    - description: Check storage provisioner status
+      script:
+        content: |
+          echo "=== Storage Classes ==="
+          kubectl get storageclass -o wide
+          echo ""
+          echo "=== Local Path Provisioner Pods ==="
+          kubectl get pods -n kube-system -l app=local-path-provisioner -o wide
+          echo ""
+          echo "=== Local Path Provisioner Logs ==="
+          kubectl logs -n kube-system -l app=local-path-provisioner --tail=100 --prefix=true
+
+    - description: Check node resources and disk space
+      script:
+        content: |
+          echo "=== Node Resources ==="
+          kubectl describe nodes
+          echo ""
+          echo "=== Node Disk Usage (via docker) ==="
+          docker exec k3d-k3s-default-server-0 df -h
+
+    - description: Check all PVs and their status
+      script:
+        content: kubectl get pv -o wide
+
+    - description: Check for stuck PVCs across all namespaces
+      script:
+        content: kubectl get pvc --all-namespaces -o wide