From 4c7cfe8f452ac2a04a46f46a8872229e137c800b Mon Sep 17 00:00:00 2001 From: vagrawal-newrelic Date: Mon, 15 Dec 2025 15:25:24 +0530 Subject: [PATCH 1/9] fix(elasticsearch-otel): Update elasticsearch otel cluster entity summary page --- .../definition.stg.yml | 6 + .../golden_metrics.stg.yml | 4 +- .../opentelemetry_dashboard.stg.json | 343 +++++++++++++++--- .../definition.stg.yml | 40 +- ...TION-to-INFRA-ELASTICSEARCHCLUSTER.stg.yml | 32 -- 5 files changed, 341 insertions(+), 84 deletions(-) delete mode 100644 relationships/synthesis/APM-APPLICATION-to-INFRA-ELASTICSEARCHCLUSTER.stg.yml diff --git a/entity-types/infra-elasticsearchcluster/definition.stg.yml b/entity-types/infra-elasticsearchcluster/definition.stg.yml index c51f603dbd..3e1356d7fd 100644 --- a/entity-types/infra-elasticsearchcluster/definition.stg.yml +++ b/entity-types/infra-elasticsearchcluster/definition.stg.yml @@ -24,6 +24,8 @@ synthesis: value: "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/elasticsearchreceiver" - attribute: elasticsearch.cluster.name present: true + - attribute: elasticsearch.node.name + present: false tags: otel.library.name: entityTagName: instrumentation.name @@ -42,6 +44,8 @@ synthesis: value: "otelcol/elasticsearchreceiver" - attribute: elasticsearch.cluster.name present: true + - attribute: elasticsearch.node.name + present: false tags: otel.library.name: entityTagName: instrumentation.name @@ -60,6 +64,8 @@ synthesis: value: 'api.logs.otlp' - attribute: elasticsearch.cluster.name present: true + - attribute: elasticsearch.node.name + present: false tags: instrumentation.provider: entityTagName: instrumentation.name diff --git a/entity-types/infra-elasticsearchcluster/golden_metrics.stg.yml b/entity-types/infra-elasticsearchcluster/golden_metrics.stg.yml index b61cf82c4d..ebf8e1733c 100644 --- a/entity-types/infra-elasticsearchcluster/golden_metrics.stg.yml +++ b/entity-types/infra-elasticsearchcluster/golden_metrics.stg.yml @@ -1,10 +1,10 @@ clusterHealth: title: Cluster Health - displayAsValue: true queries: opentelemetry: - select: latest(status) + select: latest(elasticsearch.cluster.health) from: Metric + facet: status eventId: entity.guid eventName: entity.name newRelic: diff --git a/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json b/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json index 456f0885b7..83e6ab2e25 100644 --- a/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json +++ b/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json @@ -7,110 +7,222 @@ "description": null, "widgets": [ { - "title": "Cluster Status", + "title": "Data Nodes", "layout": { "column": 1, "row": 1, - "width": 4, - "height": 3 + "width": 3, + "height": 1 }, "visualization": { "id": "viz.billboard" }, "rawConfiguration": { + "chartStyles": { + "lineInterpolation": "linear" + }, + "facet": { + "showOtherSeries": false + }, "nrqlQueries": [ { "accountId": 0, - "query": "FROM Metric SELECT latest(status)" + "query": "FROM Metric SELECT latest(elasticsearch.cluster.data_nodes) AS 'Data Nodes'" } - ] + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholdsWithSeriesOverrides": { + "thresholds": [ + { + "from": 0, + "severity": "success" + } + ] + } } }, { - "title": "Node Count", + "title": "Total Nodes", "layout": { - "column": 5, + "column": 4, "row": 1, - "width": 4, - "height": 3 + "width": 3, + "height": 1 }, "visualization": { - "id": "viz.table" + "id": "viz.billboard" }, "rawConfiguration": { + "chartStyles": { + "lineInterpolation": "linear" + }, + "facet": { + "showOtherSeries": false + }, "nrqlQueries": [ { "accountId": 0, - "query": "SELECT \n latest(elasticsearch.cluster.nodes) AS 'Total Nodes', \n latest(elasticsearch.cluster.data_nodes) AS 'Data Nodes' \nFROM Metric" + "query": "FROM Metric SELECT latest(elasticsearch.cluster.nodes) AS 'Total Nodes'" } - ] + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholdsWithSeriesOverrides": { + "thresholds": [ + { + "from": 0, + "severity": "success" + } + ] + } } }, { - "title": "Pending Task", + "title": "In-Flight Fetches", "layout": { - "column": 9, + "column": 7, "row": 1, - "width": 4, - "height": 3 + "width": 3, + "height": 1 }, "visualization": { - "id": "viz.line" + "id": "viz.billboard" }, "rawConfiguration": { + "chartStyles": { + "lineInterpolation": "linear" + }, + "facet": { + "showOtherSeries": false + }, "nrqlQueries": [ { "accountId": 0, - "query": "SELECT latest(elasticsearch.cluster.pending_tasks) \nFROM Metric\nTIMESERIES" + "query": "FROM Metric SELECT latest(elasticsearch.cluster.in_flight_fetch) AS 'In-Flight Fetches'" } - ] + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholdsWithSeriesOverrides": { + "thresholds": [ + { + "from": 0, + "severity": "success" + } + ] + } } }, { - "title": "Shards", + "title": "Current Pending Task", + "layout": { + "column": 10, + "row": 1, + "width": 3, + "height": 1 + }, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "chartStyles": { + "lineInterpolation": "linear" + }, + "facet": { + "showOtherSeries": false + }, + "nrqlQueries": [ + { + "accountId": 0, + "query": "FROM Metric SELECT latest(elasticsearch.cluster.pending_tasks) AS 'Pending Tasks'" + } + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholdsWithSeriesOverrides": { + "thresholds": [ + { + "from": 0, + "severity": "success" + } + ] + } + } + }, + { + "title": "Current Cluster Health", "layout": { "column": 1, - "row": 4, + "row": 2, "width": 4, "height": 3 }, "visualization": { - "id": "viz.pie" + "id": "viz.billboard" }, "rawConfiguration": { + "chartStyles": { + "lineInterpolation": "linear" + }, + "dataFormatters": [], + "facet": { + "showOtherSeries": false + }, "nrqlQueries": [ { "accountId": 0, - "query": "SELECT latest(elasticsearch.cluster.shards) \nFROM Metric\nFACET state" + "query": "FROM Metric SELECT \n IF(latest(status) = 'green', '🟢 GREEN',\n IF(latest(status) = 'yellow', '🟡 YELLOW',\n '🔴 RED'\n)) AS 'Cluster Health'" } - ] + ], + "platformOptions": { + "ignoreTimeRange": false + } } }, { - "title": "Document count", + "title": "Shards", "layout": { "column": 5, - "row": 4, + "row": 2, "width": 4, "height": 3 }, "visualization": { - "id": "viz.line" + "id": "viz.pie" }, "rawConfiguration": { + "chartStyles": { + "gradient": { + "enabled": false + } + }, + "facet": { + "showOtherSeries": true + }, + "legend": { + "enabled": true + }, "nrqlQueries": [ { "accountId": 0, - "query": "SELECT latest(elasticsearch.index.documents) AS 'Total Documents in cluster'\nFROM Metric \nWHERE elasticsearch.index.name = '_all' AND aggregation = 'primary_shards' TIMESERIES " + "query": "SELECT latest(elasticsearch.cluster.shards) \nFROM Metric\nFACET state" } - ] + ], + "platformOptions": { + "ignoreTimeRange": false + } } }, { - "title": "Indexing & Search Rate(op/sec)", + "title": "Document count", "layout": { "column": 9, - "row": 4, + "row": 2, "width": 4, "height": 3 }, @@ -118,75 +230,208 @@ "id": "viz.line" }, "rawConfiguration": { + "chartStyles": { + "lineInterpolation": "linear" + }, + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "markers": { + "displayedTypes": { + "criticalViolations": false, + "deployments": true, + "relatedDeployments": true, + "warningViolations": false + } + }, "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(elasticsearch.node.operations.completed), 1 second) FROM Metric WHERE operation IN ('index', 'query') FACET operation TIMESERIES 1 minute SLIDE BY 5 minute" + "query": "SELECT latest(elasticsearch.index.documents) AS 'Total Documents in cluster'\nFROM Metric \nWHERE elasticsearch.index.name = '_all' AND aggregation = 'primary_shards' TIMESERIES AUTO" } - ] + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } } }, { - "title": "Total Size on Disk (GB)", + "title": "Pending Task", "layout": { "column": 1, - "row": 7, + "row": 5, "width": 4, "height": 3 }, "visualization": { - "id": "viz.billboard" + "id": "viz.line" }, "rawConfiguration": { + "chartStyles": { + "lineInterpolation": "linear" + }, + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "markers": { + "displayedTypes": { + "criticalViolations": false, + "deployments": true, + "relatedDeployments": true, + "warningViolations": false + } + }, "nrqlQueries": [ { "accountId": 0, - "query": "SELECT sum(elasticsearch.node.shards.size) / 1073741824 as 'Disk Used(GB)' FROM Metric" + "query": "SELECT latest(elasticsearch.cluster.pending_tasks) \nFROM Metric\nTIMESERIES AUTO" } - ] + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } } }, { - "title": "Avg OS CPU Used (%)", + "title": "Cluster State Update Time", "layout": { "column": 5, - "row": 7, + "row": 5, "width": 4, "height": 3 }, "visualization": { - "id": "viz.billboard" + "id": "viz.line" }, "rawConfiguration": { + "chartStyles": { + "lineInterpolation": "linear" + }, + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "markers": { + "displayedTypes": { + "criticalViolations": false, + "deployments": true, + "relatedDeployments": true, + "warningViolations": false + } + }, "nrqlQueries": [ { "accountId": 0, - "query": "SELECT average(elasticsearch.os.cpu.usage) as 'Avg OS CPU Used (%)' FROM Metric" + "query": "SELECT rate(sum(elasticsearch.cluster.state_update.time), 1 minute) / 1000 AS 'Cluster State Update Time (s/min)' FROM Metric TIMESERIES AUTO" } - ] + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } } }, { - "title": "Total JVM Heap Used (GB)", + "title": "Cluster Health Status", "layout": { "column": 9, - "row": 7, + "row": 5, "width": 4, "height": 3 }, "visualization": { - "id": "viz.billboard" + "id": "viz.line" }, "rawConfiguration": { + "chartStyles": { + "lineInterpolation": "linear" + }, + "colors": { + "seriesOverrides": [ + { + "color": "#f00f0f", + "seriesName": "red" + }, + { + "color": "#7d7373", + "seriesName": "green" + }, + { + "color": "#63e10e", + "seriesName": "yellow" + } + ] + }, + "facet": { + "showOtherSeries": false + }, + "legend": { + "enabled": true + }, + "markers": { + "displayedTypes": { + "criticalViolations": false, + "deployments": true, + "relatedDeployments": true, + "warningViolations": false + } + }, "nrqlQueries": [ { "accountId": 0, - "query": "SELECT sum(jvm.memory.heap.used) / 1073741824 as 'Total JVM Heap Used (GB)' FROM Metric" + "query": "FROM Metric SELECT latest(elasticsearch.cluster.health) FACET status TIMESERIES AUTO" } - ] + ], + "platformOptions": { + "ignoreTimeRange": false + }, + "thresholds": { + "isLabelVisible": true + }, + "yAxisLeft": { + "zero": true + }, + "yAxisRight": { + "zero": true + } } } ] } - ] + ], + "variables": [] } \ No newline at end of file diff --git a/entity-types/infra-elasticsearchnode/definition.stg.yml b/entity-types/infra-elasticsearchnode/definition.stg.yml index 3780801cae..a7af59316e 100644 --- a/entity-types/infra-elasticsearchnode/definition.stg.yml +++ b/entity-types/infra-elasticsearchnode/definition.stg.yml @@ -20,7 +20,12 @@ synthesis: - elasticsearch.node.name - elasticsearch.cluster.name - host.id - name: elasticsearch.node.name + compositeName: + fragments: + - value: "es-node=" + - attribute: elasticsearch.node.name + - value: ":es-cluster=" + - attribute: elasticsearch.cluster.name encodeIdentifierInGUID: true conditions: - attribute: eventType @@ -41,6 +46,39 @@ synthesis: host.name: otel.library.name: entityTagName: instrumentation.name + - ruleName: infra_elasticsearchnode_composite_1 + compositeIdentifier: + separator: ":" + attributes: + - elasticsearch.node.name + - elasticsearch.cluster.name + - host.id + compositeName: + fragments: + - value: "es-node=" + - attribute: elasticsearch.node.name + - value: ":es-cluster=" + - attribute: elasticsearch.cluster.name + encodeIdentifierInGUID: true + conditions: + - attribute: eventType + value: Metric + - attribute: instrumentation.provider + value: opentelemetry + # This filters only metrics coming from elasticsearch receiver, given that metrics + # could differ between different runtime receivers. + - attribute: otel.library.name + value: "otelcol/elasticsearchreceiver" + - attribute: elasticsearch.node.name + present: true + - attribute: elasticsearch.cluster.name + present: true + tags: + elasticsearch.node.name: + elasticsearch.cluster.name: + host.name: + otel.library.name: + entityTagName: instrumentation.name tags: # For OpenTelemetry telemetry.sdk.name: diff --git a/relationships/synthesis/APM-APPLICATION-to-INFRA-ELASTICSEARCHCLUSTER.stg.yml b/relationships/synthesis/APM-APPLICATION-to-INFRA-ELASTICSEARCHCLUSTER.stg.yml deleted file mode 100644 index 78dfe5d818..0000000000 --- a/relationships/synthesis/APM-APPLICATION-to-INFRA-ELASTICSEARCHCLUSTER.stg.yml +++ /dev/null @@ -1,32 +0,0 @@ -relationships: - - name: apmCallsOtelElasticsearchCluster - version: "1" - origins: - - OpenTelemetry - conditions: - - attribute: eventType - anyOf: [ "Metric" ] - - attribute: instrumentation.provider - anyOf: [ "opentelemetry" ] - - attribute: entity.type - anyOf: [ "SERVICE" ] - - attribute: es.cluster.name - present: true - relationship: - expires: PT75M - relationshipType: CALLS - source: - extractGuid: - attribute: entity.guid - target: - buildGuid: - account: - attribute: accountId - domain: - value: INFRA - type: - value: ELASTICSEARCHCLUSTER - identifier: - fragments: - - attribute: es.cluster.name - hashAlgorithm: FARM_HASH \ No newline at end of file From bc34d758faa607e090ead50614b0748d14987153 Mon Sep 17 00:00:00 2001 From: vagrawal-newrelic Date: Mon, 15 Dec 2025 15:33:25 +0530 Subject: [PATCH 2/9] fix(elasticsearch-otel): Remove empty variable block --- .../opentelemetry_dashboard.stg.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json b/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json index 83e6ab2e25..268fa9bbbc 100644 --- a/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json +++ b/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json @@ -432,6 +432,5 @@ } ] } - ], - "variables": [] + ] } \ No newline at end of file From cf6e3068150843b801ce36433a108841f9b2b33d Mon Sep 17 00:00:00 2001 From: vagrawal-newrelic Date: Mon, 15 Dec 2025 15:54:52 +0530 Subject: [PATCH 3/9] fix(es-otel): Remove unwanted dashboard options --- .../opentelemetry_dashboard.stg.json | 234 +----------------- 1 file changed, 13 insertions(+), 221 deletions(-) diff --git a/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json b/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json index 268fa9bbbc..992b6c4e00 100644 --- a/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json +++ b/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json @@ -18,29 +18,12 @@ "id": "viz.billboard" }, "rawConfiguration": { - "chartStyles": { - "lineInterpolation": "linear" - }, - "facet": { - "showOtherSeries": false - }, "nrqlQueries": [ { "accountId": 0, "query": "FROM Metric SELECT latest(elasticsearch.cluster.data_nodes) AS 'Data Nodes'" } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholdsWithSeriesOverrides": { - "thresholds": [ - { - "from": 0, - "severity": "success" - } - ] - } + ] } }, { @@ -55,29 +38,12 @@ "id": "viz.billboard" }, "rawConfiguration": { - "chartStyles": { - "lineInterpolation": "linear" - }, - "facet": { - "showOtherSeries": false - }, "nrqlQueries": [ { "accountId": 0, "query": "FROM Metric SELECT latest(elasticsearch.cluster.nodes) AS 'Total Nodes'" } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholdsWithSeriesOverrides": { - "thresholds": [ - { - "from": 0, - "severity": "success" - } - ] - } + ] } }, { @@ -92,29 +58,12 @@ "id": "viz.billboard" }, "rawConfiguration": { - "chartStyles": { - "lineInterpolation": "linear" - }, - "facet": { - "showOtherSeries": false - }, "nrqlQueries": [ { "accountId": 0, "query": "FROM Metric SELECT latest(elasticsearch.cluster.in_flight_fetch) AS 'In-Flight Fetches'" } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholdsWithSeriesOverrides": { - "thresholds": [ - { - "from": 0, - "severity": "success" - } - ] - } + ] } }, { @@ -129,29 +78,12 @@ "id": "viz.billboard" }, "rawConfiguration": { - "chartStyles": { - "lineInterpolation": "linear" - }, - "facet": { - "showOtherSeries": false - }, "nrqlQueries": [ { "accountId": 0, "query": "FROM Metric SELECT latest(elasticsearch.cluster.pending_tasks) AS 'Pending Tasks'" } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholdsWithSeriesOverrides": { - "thresholds": [ - { - "from": 0, - "severity": "success" - } - ] - } + ] } }, { @@ -166,22 +98,12 @@ "id": "viz.billboard" }, "rawConfiguration": { - "chartStyles": { - "lineInterpolation": "linear" - }, - "dataFormatters": [], - "facet": { - "showOtherSeries": false - }, "nrqlQueries": [ { "accountId": 0, "query": "FROM Metric SELECT \n IF(latest(status) = 'green', '🟢 GREEN',\n IF(latest(status) = 'yellow', '🟡 YELLOW',\n '🔴 RED'\n)) AS 'Cluster Health'" } - ], - "platformOptions": { - "ignoreTimeRange": false - } + ] } }, { @@ -196,26 +118,12 @@ "id": "viz.pie" }, "rawConfiguration": { - "chartStyles": { - "gradient": { - "enabled": false - } - }, - "facet": { - "showOtherSeries": true - }, - "legend": { - "enabled": true - }, "nrqlQueries": [ { "accountId": 0, "query": "SELECT latest(elasticsearch.cluster.shards) \nFROM Metric\nFACET state" } - ], - "platformOptions": { - "ignoreTimeRange": false - } + ] } }, { @@ -230,41 +138,12 @@ "id": "viz.line" }, "rawConfiguration": { - "chartStyles": { - "lineInterpolation": "linear" - }, - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "markers": { - "displayedTypes": { - "criticalViolations": false, - "deployments": true, - "relatedDeployments": true, - "warningViolations": false - } - }, "nrqlQueries": [ { "accountId": 0, "query": "SELECT latest(elasticsearch.index.documents) AS 'Total Documents in cluster'\nFROM Metric \nWHERE elasticsearch.index.name = '_all' AND aggregation = 'primary_shards' TIMESERIES AUTO" } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } + ] } }, { @@ -279,41 +158,12 @@ "id": "viz.line" }, "rawConfiguration": { - "chartStyles": { - "lineInterpolation": "linear" - }, - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "markers": { - "displayedTypes": { - "criticalViolations": false, - "deployments": true, - "relatedDeployments": true, - "warningViolations": false - } - }, "nrqlQueries": [ { "accountId": 0, "query": "SELECT latest(elasticsearch.cluster.pending_tasks) \nFROM Metric\nTIMESERIES AUTO" } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } + ] } }, { @@ -328,41 +178,12 @@ "id": "viz.line" }, "rawConfiguration": { - "chartStyles": { - "lineInterpolation": "linear" - }, - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "markers": { - "displayedTypes": { - "criticalViolations": false, - "deployments": true, - "relatedDeployments": true, - "warningViolations": false - } - }, "nrqlQueries": [ { "accountId": 0, "query": "SELECT rate(sum(elasticsearch.cluster.state_update.time), 1 minute) / 1000 AS 'Cluster State Update Time (s/min)' FROM Metric TIMESERIES AUTO" } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } + ] } }, { @@ -377,57 +198,28 @@ "id": "viz.line" }, "rawConfiguration": { - "chartStyles": { - "lineInterpolation": "linear" - }, "colors": { "seriesOverrides": [ { - "color": "#f00f0f", + "color": "#FF0000", "seriesName": "red" }, { - "color": "#7d7373", + "color": "#008000", "seriesName": "green" }, { - "color": "#63e10e", + "color": "#FFFF00", "seriesName": "yellow" } ] }, - "facet": { - "showOtherSeries": false - }, - "legend": { - "enabled": true - }, - "markers": { - "displayedTypes": { - "criticalViolations": false, - "deployments": true, - "relatedDeployments": true, - "warningViolations": false - } - }, "nrqlQueries": [ { "accountId": 0, "query": "FROM Metric SELECT latest(elasticsearch.cluster.health) FACET status TIMESERIES AUTO" } - ], - "platformOptions": { - "ignoreTimeRange": false - }, - "thresholds": { - "isLabelVisible": true - }, - "yAxisLeft": { - "zero": true - }, - "yAxisRight": { - "zero": true - } + ] } } ] From 9c6f710e4fa7f92656bb2fc1ac96d7762fb0615b Mon Sep 17 00:00:00 2001 From: vagrawal-newrelic Date: Wed, 17 Dec 2025 09:04:22 +0530 Subject: [PATCH 4/9] fix(es-otel): Update elasticsearch otel experience --- .../golden_metrics.stg.yml | 16 +++++---- .../opentelemetry_dashboard.stg.json | 6 ++-- .../definition.stg.yml | 4 +++ .../golden_metrics.stg.yml | 21 ++++++----- .../opentelemetry_dashboard.stg.json | 4 +-- ...VICE-to-INFRA-ELASTICSEARCHCLUSTER.stg.yml | 32 +++++++++++++++++ ...NTAINER-to-INFRA-ELASTICSEARCHNODE.stg.yml | 36 +++++++++++++++++++ .../INFRA-HOST-to-ELASTICSEARCHNODE.stg.yml | 6 +++- 8 files changed, 105 insertions(+), 20 deletions(-) create mode 100644 relationships/synthesis/EXT-SERVICE-to-INFRA-ELASTICSEARCHCLUSTER.stg.yml create mode 100644 relationships/synthesis/INFRA-DOCKER-CONTAINER-to-INFRA-ELASTICSEARCHNODE.stg.yml diff --git a/entity-types/infra-elasticsearchcluster/golden_metrics.stg.yml b/entity-types/infra-elasticsearchcluster/golden_metrics.stg.yml index ebf8e1733c..b3e16465ba 100644 --- a/entity-types/infra-elasticsearchcluster/golden_metrics.stg.yml +++ b/entity-types/infra-elasticsearchcluster/golden_metrics.stg.yml @@ -4,6 +4,7 @@ clusterHealth: opentelemetry: select: latest(elasticsearch.cluster.health) from: Metric + where: metricName = 'elasticsearch.cluster.health' facet: status eventId: entity.guid eventName: entity.name @@ -18,6 +19,7 @@ clusterDataNode: opentelemetry: select: average(elasticsearch.cluster.data_nodes) from: Metric + where: metricName = 'elasticsearch.cluster.data_nodes' eventId: entity.guid eventName: entity.name newRelic: @@ -31,6 +33,7 @@ clusterNode: opentelemetry: select: average(elasticsearch.cluster.nodes) from: Metric + where: metricName = 'elasticsearch.cluster.nodes' eventId: entity.guid eventName: entity.name newRelic: @@ -44,7 +47,7 @@ clusterActiveShard: opentelemetry: select: average(elasticsearch.cluster.shards) from: Metric - where: state='active' + where: metricName = 'elasticsearch.cluster.shards' AND state='active' eventId: entity.guid eventName: entity.name clusterInitializingShard: @@ -53,7 +56,7 @@ clusterInitializingShard: opentelemetry: select: average(elasticsearch.cluster.shards) from: Metric - where: state='initializing' + where: metricName = 'elasticsearch.cluster.shards' AND state='initializing' eventId: entity.guid eventName: entity.name clusterPrimaryActiveShard: @@ -62,7 +65,7 @@ clusterPrimaryActiveShard: opentelemetry: select: average(elasticsearch.cluster.shards) from: Metric - where: state='active_primary' + where: metricName = 'elasticsearch.cluster.shards' AND state='active_primary' eventId: entity.guid eventName: entity.name clusterRelocatingShard: @@ -71,7 +74,7 @@ clusterRelocatingShard: opentelemetry: select: average(elasticsearch.cluster.shards) from: Metric - where: state='relocating' + where: metricName = 'elasticsearch.cluster.shards' AND state='relocating' eventId: entity.guid eventName: entity.name clusterUnassignedShard: @@ -80,7 +83,7 @@ clusterUnassignedShard: opentelemetry: select: average(elasticsearch.cluster.shards) from: Metric - where: state='unassigned' + where: metricName = 'elasticsearch.cluster.shards' AND state='unassigned' eventId: entity.guid eventName: entity.name newRelic: @@ -89,10 +92,11 @@ clusterUnassignedShard: eventId: entityGuid eventName: entityName clusterPendingTask: - title: Cluster Pending Task + title: Cluster Pending Tasks queries: opentelemetry: select: latest(elasticsearch.cluster.pending_tasks) from: Metric + where: metricName = 'elasticsearch.cluster.pending_tasks' eventId: entity.guid eventName: entity.name \ No newline at end of file diff --git a/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json b/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json index 992b6c4e00..25f06749da 100644 --- a/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json +++ b/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json @@ -67,7 +67,7 @@ } }, { - "title": "Current Pending Task", + "title": "Pending Tasks", "layout": { "column": 10, "row": 1, @@ -87,7 +87,7 @@ } }, { - "title": "Current Cluster Health", + "title": "Cluster Health", "layout": { "column": 1, "row": 2, @@ -147,7 +147,7 @@ } }, { - "title": "Pending Task", + "title": "Pending Tasks", "layout": { "column": 1, "row": 5, diff --git a/entity-types/infra-elasticsearchnode/definition.stg.yml b/entity-types/infra-elasticsearchnode/definition.stg.yml index a7af59316e..55dc7db8f4 100644 --- a/entity-types/infra-elasticsearchnode/definition.stg.yml +++ b/entity-types/infra-elasticsearchnode/definition.stg.yml @@ -40,6 +40,8 @@ synthesis: present: true - attribute: elasticsearch.cluster.name present: true + - attribute: host.id + present: true tags: elasticsearch.node.name: elasticsearch.cluster.name: @@ -73,6 +75,8 @@ synthesis: present: true - attribute: elasticsearch.cluster.name present: true + - attribute: host.id + present: true tags: elasticsearch.node.name: elasticsearch.cluster.name: diff --git a/entity-types/infra-elasticsearchnode/golden_metrics.stg.yml b/entity-types/infra-elasticsearchnode/golden_metrics.stg.yml index 76d847bc3b..1a7be06628 100644 --- a/entity-types/infra-elasticsearchnode/golden_metrics.stg.yml +++ b/entity-types/infra-elasticsearchnode/golden_metrics.stg.yml @@ -12,7 +12,7 @@ activeSearches: from: Metric where: operation='query' AND metricName = 'elasticsearch.node.operations.current' eventId: entity.guid - eventName: elasticsearch.node.name + eventName: entity.name missingDocumentRequests: title: Missing document requests unit: COUNT @@ -23,11 +23,11 @@ missingDocumentRequests: eventId: entityGuid eventName: entityName opentelemetry: - select: average(getField(elasticsearch.node.operations.get.completed, cumulative)) + select: average(elasticsearch.node.operations.get.completed) from: Metric where: result='miss' AND metricName = 'elasticsearch.node.operations.get.completed' eventId: entity.guid - eventName: elasticsearch.node.name + eventName: entity.name fileStoreIOOperations: title: File store I/O operations unit: COUNT @@ -37,7 +37,12 @@ fileStoreIOOperations: from: ElasticsearchNodeSample eventId: entityGuid eventName: entityName - # This metric is not available in the current OTEL Elasticsearch receiver + opentelemetry: + select: average(elasticsearch.node.disk.io.read) + from: Metric + where: metricName = 'elasticsearch.node.disk.io.read' + eventId: entity.guid + eventName: entity.name activeMerges: queries: newRelic: @@ -46,11 +51,11 @@ activeMerges: from: ElasticsearchNodeSample select: max(`merges.currentActive`) opentelemetry: - select: max(`elasticsearch.node.operations.current`) + select: max(elasticsearch.node.operations.current) from: Metric where: metricName = 'elasticsearch.node.operations.current' and operation = 'merge' eventId: entity.guid - eventName: elasticsearch.node.name + eventName: entity.name unit: COUNT title: Current active merges totalQueries: @@ -63,8 +68,8 @@ totalQueries: select: average(`queriesTotal`) from: ElasticsearchNodeSample opentelemetry: - select: average(getField(`elasticsearch.node.operations.completed`, cumulative)) + select: average(elasticsearch.node.operations.completed) from: Metric where: operation='query' AND metricName = 'elasticsearch.node.operations.completed' eventId: entity.guid - eventName: elasticsearch.node.name + eventName: entity.name diff --git a/entity-types/infra-elasticsearchnode/opentelemetry_dashboard.stg.json b/entity-types/infra-elasticsearchnode/opentelemetry_dashboard.stg.json index b6836aa168..1998bbc8f4 100644 --- a/entity-types/infra-elasticsearchnode/opentelemetry_dashboard.stg.json +++ b/entity-types/infra-elasticsearchnode/opentelemetry_dashboard.stg.json @@ -15,7 +15,7 @@ "title" : "Running GET requests and Missing requests", "rawConfiguration" : { "nrqlQueries" : [ { - "query" : "SELECT filter(average(`elasticsearch.node.operations.current`), where operation='get' and metricName = 'elasticsearch.node.operations.current' ) AS `Running`, filter(average(getField(elasticsearch.node.operations.get.completed, cumulative)), where result='miss' and metricName = 'elasticsearch.node.operations.get.completed' ) AS 'Missing' FROM Metric TIMESERIES AUTO", + "query" : "SELECT filter(average(`elasticsearch.node.operations.current`), where operation='get' and metricName = 'elasticsearch.node.operations.current' ) AS `Running`, filter(average(`elasticsearch.node.operations.get.completed`), where result='miss' and metricName = 'elasticsearch.node.operations.get.completed' ) AS 'Missing' FROM Metric TIMESERIES AUTO", "accountId": 0} ] } }, { @@ -79,7 +79,7 @@ "title" : "Time spent indexing and deleting documents (secs)", "rawConfiguration" : { "nrqlQueries" : [ { - "query" : "SELECT average(getField(elasticsearch.node.operations.time, cumulative))/1000 FROM Metric WHERE metricName = 'elasticsearch.node.operations.time' facet operation in ('index', 'delete') TIMESERIES AUTO", + "query" : "SELECT average(elasticsearch.node.operations.time)/1000 FROM Metric WHERE metricName = 'elasticsearch.node.operations.time' facet operation in ('index', 'delete') TIMESERIES AUTO", "accountId": 0} ] } }, { diff --git a/relationships/synthesis/EXT-SERVICE-to-INFRA-ELASTICSEARCHCLUSTER.stg.yml b/relationships/synthesis/EXT-SERVICE-to-INFRA-ELASTICSEARCHCLUSTER.stg.yml new file mode 100644 index 0000000000..78dfe5d818 --- /dev/null +++ b/relationships/synthesis/EXT-SERVICE-to-INFRA-ELASTICSEARCHCLUSTER.stg.yml @@ -0,0 +1,32 @@ +relationships: + - name: apmCallsOtelElasticsearchCluster + version: "1" + origins: + - OpenTelemetry + conditions: + - attribute: eventType + anyOf: [ "Metric" ] + - attribute: instrumentation.provider + anyOf: [ "opentelemetry" ] + - attribute: entity.type + anyOf: [ "SERVICE" ] + - attribute: es.cluster.name + present: true + relationship: + expires: PT75M + relationshipType: CALLS + source: + extractGuid: + attribute: entity.guid + target: + buildGuid: + account: + attribute: accountId + domain: + value: INFRA + type: + value: ELASTICSEARCHCLUSTER + identifier: + fragments: + - attribute: es.cluster.name + hashAlgorithm: FARM_HASH \ No newline at end of file diff --git a/relationships/synthesis/INFRA-DOCKER-CONTAINER-to-INFRA-ELASTICSEARCHNODE.stg.yml b/relationships/synthesis/INFRA-DOCKER-CONTAINER-to-INFRA-ELASTICSEARCHNODE.stg.yml new file mode 100644 index 0000000000..e5935da8f6 --- /dev/null +++ b/relationships/synthesis/INFRA-DOCKER-CONTAINER-to-INFRA-ELASTICSEARCHNODE.stg.yml @@ -0,0 +1,36 @@ +relationships: + - name: containerHostsOtelElasticSearchNode + version: "1" + origins: + - OpenTelemetry + conditions: + - attribute: eventType + anyOf: [ "Metric" ] + - attribute: instrumentation.provider + anyOf: [ "opentelemetry" ] + - attribute: elasticsearch.node.name + present: true + - attribute: entity.type + anyOf: ["ELASTICSEARCHNODE"] + - attribute: container.id + present: true + relationship: + expires: PT75M + relationshipType: HOSTS + source: + buildGuid: + account: + attribute: accountId + domain: + value: INFRA + type: + value: CONTAINER + identifier: + fragments: + - attribute: container.id + hashAlgorithm: FARM_HASH + target: + extractGuid: + attribute: entity.guid + entityType: + attribute: entity.type \ No newline at end of file diff --git a/relationships/synthesis/INFRA-HOST-to-ELASTICSEARCHNODE.stg.yml b/relationships/synthesis/INFRA-HOST-to-ELASTICSEARCHNODE.stg.yml index 3d61dcac5e..70b7aae83c 100644 --- a/relationships/synthesis/INFRA-HOST-to-ELASTICSEARCHNODE.stg.yml +++ b/relationships/synthesis/INFRA-HOST-to-ELASTICSEARCHNODE.stg.yml @@ -6,12 +6,16 @@ relationships: conditions: - attribute: eventType anyOf: [ "Metric" ] + - attribute: instrumentation.provider + anyOf: [ "opentelemetry" ] - attribute: elasticsearch.node.name present: true - attribute: entity.type anyOf: ["ELASTICSEARCHNODE"] - attribute: host.id - present: true + present: true + - attribute: container.id + present: false relationship: expires: PT75M relationshipType: HOSTS From 5c364e2e1dffeda56d4df46a6c64e28420a44308 Mon Sep 17 00:00:00 2001 From: vagrawal-newrelic Date: Wed, 17 Dec 2025 09:08:25 +0530 Subject: [PATCH 5/9] fix(es-otel): Rename docker es relationship file name --- ... => INFRA-DOCKER_CONTAINER-to-INFRA-ELASTICSEARCHNODE.stg.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename relationships/synthesis/{INFRA-DOCKER-CONTAINER-to-INFRA-ELASTICSEARCHNODE.stg.yml => INFRA-DOCKER_CONTAINER-to-INFRA-ELASTICSEARCHNODE.stg.yml} (100%) diff --git a/relationships/synthesis/INFRA-DOCKER-CONTAINER-to-INFRA-ELASTICSEARCHNODE.stg.yml b/relationships/synthesis/INFRA-DOCKER_CONTAINER-to-INFRA-ELASTICSEARCHNODE.stg.yml similarity index 100% rename from relationships/synthesis/INFRA-DOCKER-CONTAINER-to-INFRA-ELASTICSEARCHNODE.stg.yml rename to relationships/synthesis/INFRA-DOCKER_CONTAINER-to-INFRA-ELASTICSEARCHNODE.stg.yml From b7e43b849d288b4ebde172ca9ed9d8715930cf13 Mon Sep 17 00:00:00 2001 From: vagrawal-newrelic Date: Wed, 17 Dec 2025 20:12:36 +0530 Subject: [PATCH 6/9] fix(kafka-otel): Update kafka golden metric --- .../infra-kafkacluster/golden_metrics.stg.yml | 16 ++++++++-------- ...INFRA-KAFKABROKER-to-INFRA-KAFKATOPIC.stg.yml | 4 +++- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/entity-types/infra-kafkacluster/golden_metrics.stg.yml b/entity-types/infra-kafkacluster/golden_metrics.stg.yml index 8c88761e4c..ecb6ecfc1f 100644 --- a/entity-types/infra-kafkacluster/golden_metrics.stg.yml +++ b/entity-types/infra-kafkacluster/golden_metrics.stg.yml @@ -3,12 +3,12 @@ brokerCount: unit: COUNT queries: newRelic: - select: latest(cluster.activeBrokerCount) + select: max(cluster.activeBrokerCount) from: KafkaClusterSample eventId: entityGuid eventName: entityName opentelemetry: - select: latest(kafka.brokers) + select: max(kafka.brokers) from: Metric where: metricName='kafka.brokers' eventId: entity.guid @@ -18,14 +18,14 @@ TopicCount: unit: COUNT queries: newRelic: - select: latest(cluster.globalTopicCount) + select: max(cluster.globalTopicCount) from: KafkaClusterSample eventId: entityGuid eventName: entityName opentelemetry: - select: latest(kafka.cluster.topic.count) + select: max(kafka.cluster.topic.count) from: Metric - where: metricName='kafka.cluster.topic.count' and kafka.cluster.topic.count > 0 + where: metricName='kafka.cluster.topic.count' eventId: entity.guid partitionCount: @@ -33,14 +33,14 @@ partitionCount: unit: COUNT queries: newRelic: - select: latest(cluster.globalPartitionCount) + select: max(cluster.globalPartitionCount) from: KafkaClusterSample eventId: entityGuid eventName: entityName opentelemetry: - select: latest(kafka.cluster.partition.count) + select: max(kafka.cluster.partition.count) from: Metric - where: metricName='kafka.cluster.partition.count' and kafka.cluster.partition.count > 0 + where: metricName='kafka.cluster.partition.count' eventId: entity.guid offlinePartitions: diff --git a/relationships/synthesis/INFRA-KAFKABROKER-to-INFRA-KAFKATOPIC.stg.yml b/relationships/synthesis/INFRA-KAFKABROKER-to-INFRA-KAFKATOPIC.stg.yml index 12a6f0562d..fcd37519bc 100644 --- a/relationships/synthesis/INFRA-KAFKABROKER-to-INFRA-KAFKATOPIC.stg.yml +++ b/relationships/synthesis/INFRA-KAFKABROKER-to-INFRA-KAFKATOPIC.stg.yml @@ -82,6 +82,8 @@ relationships: anyOf: [ "KAFKATOPIC" ] - attribute: broker.id present: true + - attribute: kafka.cluster.name + present: true relationship: expires: PT75M relationshipType: MANAGES @@ -97,7 +99,7 @@ relationships: fragments: - attribute: broker.id - value: ":" - - attribute: cluster.name + - attribute: kafka.cluster.name hashAlgorithm: FARM_HASH target: extractGuid: From 424f58c656962b9421d947d99c903bf170e056b5 Mon Sep 17 00:00:00 2001 From: vagrawal-newrelic Date: Wed, 17 Dec 2025 20:20:56 +0530 Subject: [PATCH 7/9] fix(kafka-otel): Update kafka relationship in staging --- .../INFRA-HOST-to-INFRA-KAFKABROKER.stg.yml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/relationships/synthesis/INFRA-HOST-to-INFRA-KAFKABROKER.stg.yml b/relationships/synthesis/INFRA-HOST-to-INFRA-KAFKABROKER.stg.yml index 1a5e40d330..e66d700b34 100644 --- a/relationships/synthesis/INFRA-HOST-to-INFRA-KAFKABROKER.stg.yml +++ b/relationships/synthesis/INFRA-HOST-to-INFRA-KAFKABROKER.stg.yml @@ -205,3 +205,31 @@ relationships: entityType: value: KAFKABROKER + # Host measures Kafka broker with OpenTelemetry when broker.endpoint is not present + - name: hostMonitorsOtelKafkaBroker2 + version: "1" + origins: + - OpenTelemetry + conditions: + - attribute: eventType + anyOf: [ "Metric" ] + - attribute: instrumentation.provider + anyOf: [ "opentelemetry" ] + - attribute: host.id + present: true + - attribute: broker.endpoint + present: false + relationship: + expires: PT75M + relationshipType: MEASURES + source: + lookupGuid: + candidateCategory: HOST + fields: + - field: hostId + attribute: host.id + target: + extractGuid: + attribute: entity.guid + entityType: + value: KAFKABROKER \ No newline at end of file From 0b7e18b32f21429cde434b68a8f9d95c0283d7dc Mon Sep 17 00:00:00 2001 From: vagrawal-newrelic Date: Wed, 17 Dec 2025 21:12:48 +0530 Subject: [PATCH 8/9] fix(kafka-otel): Update kafka cluster golden metrics --- entity-types/infra-kafkacluster/golden_metrics.stg.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/entity-types/infra-kafkacluster/golden_metrics.stg.yml b/entity-types/infra-kafkacluster/golden_metrics.stg.yml index ecb6ecfc1f..f249d2ed43 100644 --- a/entity-types/infra-kafkacluster/golden_metrics.stg.yml +++ b/entity-types/infra-kafkacluster/golden_metrics.stg.yml @@ -25,7 +25,7 @@ TopicCount: opentelemetry: select: max(kafka.cluster.topic.count) from: Metric - where: metricName='kafka.cluster.topic.count' + where: metricName='kafka.cluster.topic.count' and getField(kafka.cluster.topic.count , 'count') > 0 eventId: entity.guid partitionCount: @@ -40,7 +40,7 @@ partitionCount: opentelemetry: select: max(kafka.cluster.partition.count) from: Metric - where: metricName='kafka.cluster.partition.count' + where: metricName='kafka.cluster.partition.count' and getField(kafka.cluster.partition.count , 'count') > 0 eventId: entity.guid offlinePartitions: From 9ec05ba4d205672063cae75baeb350d425c23cdf Mon Sep 17 00:00:00 2001 From: vagrawal-newrelic Date: Wed, 17 Dec 2025 21:17:12 +0530 Subject: [PATCH 9/9] fix(kafka-otel): Revert previous max function to latest --- .../infra-kafkacluster/golden_metrics.stg.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/entity-types/infra-kafkacluster/golden_metrics.stg.yml b/entity-types/infra-kafkacluster/golden_metrics.stg.yml index f249d2ed43..4c13e89a83 100644 --- a/entity-types/infra-kafkacluster/golden_metrics.stg.yml +++ b/entity-types/infra-kafkacluster/golden_metrics.stg.yml @@ -3,12 +3,12 @@ brokerCount: unit: COUNT queries: newRelic: - select: max(cluster.activeBrokerCount) + select: latest(cluster.activeBrokerCount) from: KafkaClusterSample eventId: entityGuid eventName: entityName opentelemetry: - select: max(kafka.brokers) + select: latest(kafka.brokers) from: Metric where: metricName='kafka.brokers' eventId: entity.guid @@ -18,12 +18,12 @@ TopicCount: unit: COUNT queries: newRelic: - select: max(cluster.globalTopicCount) + select: latest(cluster.globalTopicCount) from: KafkaClusterSample eventId: entityGuid eventName: entityName opentelemetry: - select: max(kafka.cluster.topic.count) + select: latest(kafka.cluster.topic.count) from: Metric where: metricName='kafka.cluster.topic.count' and getField(kafka.cluster.topic.count , 'count') > 0 eventId: entity.guid @@ -33,12 +33,12 @@ partitionCount: unit: COUNT queries: newRelic: - select: max(cluster.globalPartitionCount) + select: latest(cluster.globalPartitionCount) from: KafkaClusterSample eventId: entityGuid eventName: entityName opentelemetry: - select: max(kafka.cluster.partition.count) + select: latest(kafka.cluster.partition.count) from: Metric where: metricName='kafka.cluster.partition.count' and getField(kafka.cluster.partition.count , 'count') > 0 eventId: entity.guid