Decrease frequency of writes for monitoring/analytics

Author: nwthomas

argocd/apps/logging/grafana-loki-app.yaml

@@ -18,6 +18,17 @@ spec:
             local:
               chunks_directory: /var/loki/chunks
               rules_directory: /var/loki/rules
+          limits_config:
+            ingestion_rate_mb: 10
+            ingestion_burst_size_mb: 20
+          chunk_store_config:
+            max_chunk_age: 2h
+          ingester:
+            chunk_idle_period: 30m
+            chunk_block_size: 262144
+            chunk_target_size: 1572864
+            max_chunk_age: 2h
+            flush_check_period: 30s
         persistence:
           enabled: true
           storageClassName: longhorn

argocd/apps/logging/grafana-promtail-app.yaml

@@ -14,6 +14,12 @@ spec:
         daemonset:
           tolerations:
             - operator: Exists
+        config:
+          clients:
+            - url: http://loki-gateway/loki/api/v1/push
+              batchwait: 5s
+              batchsize: 1048576
+              timeout: 30s
   destination:
     server: https://kubernetes.default.svc
     namespace: logging

helm/prometheus/templates/prometheus.yaml

@@ -12,6 +12,9 @@ spec:
   replicas: {{ .Values.replicas | default 1 }}
   retention: {{ .Values.retention | default "7d" }}
   retentionSize: {{ .Values.retentionSize | default "60GB" }}
+  walCompression: {{ .Values.walCompression | default true }}
+  scrapeInterval: {{ .Values.scrapeInterval | default "60s" }}
+  evaluationInterval: {{ .Values.evaluationInterval | default "60s" }}
   resources:
     requests:
       memory: {{ .Values.resources.requests.memory | default "400Mi" }}

helm/prometheus/values.yaml

@@ -3,6 +3,11 @@ replicas: 1
 retention: "7d"
 retentionSize: "60GB"
 
+# Write optimization settings
+walCompression: true
+scrapeInterval: "60s"
+evaluationInterval: "60s"
+
 resources:
   requests:
     memory: "400Mi"