Monitoring with Prometheus - Build Grafana Dashboards for Model

Author: ChristopherGS

packages/ml_api/api/controller.py

@@ -4,12 +4,44 @@
 
 from flask import request, jsonify, Response, current_app
 
+from gradient_boosting_model import __version__ as shadow_version
+from regression_model import __version__ as live_version
+from prometheus_client import Histogram, Gauge, Info
 from gradient_boosting_model.predict import make_prediction
 from api.persistence.data_access import PredictionPersistence, ModelType
+from api.config import APP_NAME
 
 
 _logger = logging.getLogger(__name__)
 
+PREDICTION_TRACKER = Histogram(
+    name='house_price_prediction_dollars',
+    documentation='ML Model Prediction on House Price',
+    labelnames=['app_name', 'model_name', 'model_version']
+)
+
+PREDICTION_GAUGE = Gauge(
+    name='house_price_gauge_dollars',
+    documentation='ML Model Prediction on House Price for min max calcs',
+    labelnames=['app_name', 'model_name', 'model_version']
+)
+
+PREDICTION_GAUGE.labels(
+                app_name=APP_NAME,
+                model_name=ModelType.LASSO.name,
+                model_version=live_version)
+
+MODEL_VERSIONS = Info(
+    'model_version_details',
+    'Capture model version information',
+)
+
+MODEL_VERSIONS.info({
+    'live_model': ModelType.LASSO.name,
+    'live_version': live_version,
+    'shadow_model': ModelType.GRADIENT_BOOSTING.name,
+    'shadow_version': shadow_version})
+
 
 def health():
     if request.method == "GET":
@@ -47,7 +79,18 @@ def predict():
             _logger.warning(f"errors during prediction: {result.errors}")
             return Response(json.dumps(result.errors), status=400)
 
-        # Step 4: Prepare prediction response
+        # Step 4: Monitoring
+        for _prediction in result.predictions:
+            PREDICTION_TRACKER.labels(
+                app_name=APP_NAME,
+                model_name=ModelType.LASSO.name,
+                model_version=live_version).observe(_prediction)
+            PREDICTION_GAUGE.labels(
+                app_name=APP_NAME,
+                model_name=ModelType.LASSO.name,
+                model_version=live_version).set(_prediction)
+
+        # Step 5: Prepare prediction response
         return jsonify(
             {
                 "predictions": result.predictions,

packages/ml_api/docker/config/grafana/grafana_flask_basic_dashboard_ml_api.json

@@ -15,7 +15,7 @@
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
-  "id": 2,
+  "id": 3,
   "links": [],
   "panels": [
     {
@@ -59,7 +59,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "rate(http_request_count_total{job=\"ml_api\"}[1m])",
+          "expr": "rate(http_request_count_total{job=\"ml_api\"}[5m])",
           "legendFormat": "{{app_name}} {{method}} {{endpoint}} {{http_status}}",
           "refId": "A"
         }
@@ -149,11 +149,6 @@
           "expr": "sum (rate(http_request_latency_seconds_sum{job=\"ml_api\"}[5m])) / sum (rate(http_request_latency_seconds_count{job=\"ml_api\"}[5m]))",
           "legendFormat": "Average (seconds)",
           "refId": "A"
-        },
-        {
-          "expr": "rate(http_request_latency_seconds_sum{job=\"ml_api\"}[5m]) / rate(http_request_latency_seconds_count{job=\"ml_api\"}[5m])",
-          "legendFormat": "{{endpoint}} (seconds)",
-          "refId": "B"
         }
       ],
       "thresholds": [],
@@ -205,7 +200,7 @@
     "list": []
   },
   "time": {
-    "from": "now-5m",
+    "from": "now-1h",
     "to": "now"
   },
   "timepicker": {
@@ -224,6 +219,6 @@
   },
   "timezone": "",
   "title": "Really Simple Flask Dashboard",
-  "uid": "q8vgEpLZk",
-  "version": 6
+  "uid": "q8vgEpLZl",
+  "version": 3
 }