diff --git a/entity-types/infra-elasticsearchcluster/definition.stg.yml b/entity-types/infra-elasticsearchcluster/definition.stg.yml index c51f603db..3e1356d7f 100644 --- a/entity-types/infra-elasticsearchcluster/definition.stg.yml +++ b/entity-types/infra-elasticsearchcluster/definition.stg.yml @@ -24,6 +24,8 @@ synthesis: value: "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/elasticsearchreceiver" - attribute: elasticsearch.cluster.name present: true + - attribute: elasticsearch.node.name + present: false tags: otel.library.name: entityTagName: instrumentation.name @@ -42,6 +44,8 @@ synthesis: value: "otelcol/elasticsearchreceiver" - attribute: elasticsearch.cluster.name present: true + - attribute: elasticsearch.node.name + present: false tags: otel.library.name: entityTagName: instrumentation.name @@ -60,6 +64,8 @@ synthesis: value: 'api.logs.otlp' - attribute: elasticsearch.cluster.name present: true + - attribute: elasticsearch.node.name + present: false tags: instrumentation.provider: entityTagName: instrumentation.name diff --git a/entity-types/infra-elasticsearchcluster/golden_metrics.stg.yml b/entity-types/infra-elasticsearchcluster/golden_metrics.stg.yml index b61cf82c4..b3e16465b 100644 --- a/entity-types/infra-elasticsearchcluster/golden_metrics.stg.yml +++ b/entity-types/infra-elasticsearchcluster/golden_metrics.stg.yml @@ -1,10 +1,11 @@ clusterHealth: title: Cluster Health - displayAsValue: true queries: opentelemetry: - select: latest(status) + select: latest(elasticsearch.cluster.health) from: Metric + where: metricName = 'elasticsearch.cluster.health' + facet: status eventId: entity.guid eventName: entity.name newRelic: @@ -18,6 +19,7 @@ clusterDataNode: opentelemetry: select: average(elasticsearch.cluster.data_nodes) from: Metric + where: metricName = 'elasticsearch.cluster.data_nodes' eventId: entity.guid eventName: entity.name newRelic: @@ -31,6 +33,7 @@ clusterNode: opentelemetry: select: average(elasticsearch.cluster.nodes) from: Metric + where: metricName = 'elasticsearch.cluster.nodes' eventId: entity.guid eventName: entity.name newRelic: @@ -44,7 +47,7 @@ clusterActiveShard: opentelemetry: select: average(elasticsearch.cluster.shards) from: Metric - where: state='active' + where: metricName = 'elasticsearch.cluster.shards' AND state='active' eventId: entity.guid eventName: entity.name clusterInitializingShard: @@ -53,7 +56,7 @@ clusterInitializingShard: opentelemetry: select: average(elasticsearch.cluster.shards) from: Metric - where: state='initializing' + where: metricName = 'elasticsearch.cluster.shards' AND state='initializing' eventId: entity.guid eventName: entity.name clusterPrimaryActiveShard: @@ -62,7 +65,7 @@ clusterPrimaryActiveShard: opentelemetry: select: average(elasticsearch.cluster.shards) from: Metric - where: state='active_primary' + where: metricName = 'elasticsearch.cluster.shards' AND state='active_primary' eventId: entity.guid eventName: entity.name clusterRelocatingShard: @@ -71,7 +74,7 @@ clusterRelocatingShard: opentelemetry: select: average(elasticsearch.cluster.shards) from: Metric - where: state='relocating' + where: metricName = 'elasticsearch.cluster.shards' AND state='relocating' eventId: entity.guid eventName: entity.name clusterUnassignedShard: @@ -80,7 +83,7 @@ clusterUnassignedShard: opentelemetry: select: average(elasticsearch.cluster.shards) from: Metric - where: state='unassigned' + where: metricName = 'elasticsearch.cluster.shards' AND state='unassigned' eventId: entity.guid eventName: entity.name newRelic: @@ -89,10 +92,11 @@ clusterUnassignedShard: eventId: entityGuid eventName: entityName clusterPendingTask: - title: Cluster Pending Task + title: Cluster Pending Tasks queries: opentelemetry: select: latest(elasticsearch.cluster.pending_tasks) from: Metric + where: metricName = 'elasticsearch.cluster.pending_tasks' eventId: entity.guid eventName: entity.name \ No newline at end of file diff --git a/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json b/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json index 456f0885b..25f06749d 100644 --- a/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json +++ b/entity-types/infra-elasticsearchcluster/opentelemetry_dashboard.stg.json @@ -7,12 +7,12 @@ "description": null, "widgets": [ { - "title": "Cluster Status", + "title": "Data Nodes", "layout": { "column": 1, "row": 1, - "width": 4, - "height": 3 + "width": 3, + "height": 1 }, "visualization": { "id": "viz.billboard" @@ -21,96 +21,116 @@ "nrqlQueries": [ { "accountId": 0, - "query": "FROM Metric SELECT latest(status)" + "query": "FROM Metric SELECT latest(elasticsearch.cluster.data_nodes) AS 'Data Nodes'" } ] } }, { - "title": "Node Count", + "title": "Total Nodes", "layout": { - "column": 5, + "column": 4, "row": 1, - "width": 4, - "height": 3 + "width": 3, + "height": 1 }, "visualization": { - "id": "viz.table" + "id": "viz.billboard" }, "rawConfiguration": { "nrqlQueries": [ { "accountId": 0, - "query": "SELECT \n latest(elasticsearch.cluster.nodes) AS 'Total Nodes', \n latest(elasticsearch.cluster.data_nodes) AS 'Data Nodes' \nFROM Metric" + "query": "FROM Metric SELECT latest(elasticsearch.cluster.nodes) AS 'Total Nodes'" } ] } }, { - "title": "Pending Task", + "title": "In-Flight Fetches", "layout": { - "column": 9, + "column": 7, "row": 1, - "width": 4, - "height": 3 + "width": 3, + "height": 1 }, "visualization": { - "id": "viz.line" + "id": "viz.billboard" }, "rawConfiguration": { "nrqlQueries": [ { "accountId": 0, - "query": "SELECT latest(elasticsearch.cluster.pending_tasks) \nFROM Metric\nTIMESERIES" + "query": "FROM Metric SELECT latest(elasticsearch.cluster.in_flight_fetch) AS 'In-Flight Fetches'" } ] } }, { - "title": "Shards", + "title": "Pending Tasks", + "layout": { + "column": 10, + "row": 1, + "width": 3, + "height": 1 + }, + "visualization": { + "id": "viz.billboard" + }, + "rawConfiguration": { + "nrqlQueries": [ + { + "accountId": 0, + "query": "FROM Metric SELECT latest(elasticsearch.cluster.pending_tasks) AS 'Pending Tasks'" + } + ] + } + }, + { + "title": "Cluster Health", "layout": { "column": 1, - "row": 4, + "row": 2, "width": 4, "height": 3 }, "visualization": { - "id": "viz.pie" + "id": "viz.billboard" }, "rawConfiguration": { "nrqlQueries": [ { "accountId": 0, - "query": "SELECT latest(elasticsearch.cluster.shards) \nFROM Metric\nFACET state" + "query": "FROM Metric SELECT \n IF(latest(status) = 'green', '🟢 GREEN',\n IF(latest(status) = 'yellow', '🟡 YELLOW',\n '🔴 RED'\n)) AS 'Cluster Health'" } ] } }, { - "title": "Document count", + "title": "Shards", "layout": { "column": 5, - "row": 4, + "row": 2, "width": 4, "height": 3 }, "visualization": { - "id": "viz.line" + "id": "viz.pie" }, "rawConfiguration": { "nrqlQueries": [ { "accountId": 0, - "query": "SELECT latest(elasticsearch.index.documents) AS 'Total Documents in cluster'\nFROM Metric \nWHERE elasticsearch.index.name = '_all' AND aggregation = 'primary_shards' TIMESERIES " + "query": "SELECT latest(elasticsearch.cluster.shards) \nFROM Metric\nFACET state" } ] } }, { - "title": "Indexing & Search Rate(op/sec)", + "title": "Document count", "layout": { "column": 9, - "row": 4, + "row": 2, "width": 4, "height": 3 }, @@ -121,67 +141,83 @@ "nrqlQueries": [ { "accountId": 0, - "query": "SELECT rate(sum(elasticsearch.node.operations.completed), 1 second) FROM Metric WHERE operation IN ('index', 'query') FACET operation TIMESERIES 1 minute SLIDE BY 5 minute" + "query": "SELECT latest(elasticsearch.index.documents) AS 'Total Documents in cluster'\nFROM Metric \nWHERE elasticsearch.index.name = '_all' AND aggregation = 'primary_shards' TIMESERIES AUTO" } ] } }, { - "title": "Total Size on Disk (GB)", + "title": "Pending Tasks", "layout": { "column": 1, - "row": 7, + "row": 5, "width": 4, "height": 3 }, "visualization": { - "id": "viz.billboard" + "id": "viz.line" }, "rawConfiguration": { "nrqlQueries": [ { "accountId": 0, - "query": "SELECT sum(elasticsearch.node.shards.size) / 1073741824 as 'Disk Used(GB)' FROM Metric" + "query": "SELECT latest(elasticsearch.cluster.pending_tasks) \nFROM Metric\nTIMESERIES AUTO" } ] } }, { - "title": "Avg OS CPU Used (%)", + "title": "Cluster State Update Time", "layout": { "column": 5, - "row": 7, + "row": 5, "width": 4, "height": 3 }, "visualization": { - "id": "viz.billboard" + "id": "viz.line" }, "rawConfiguration": { "nrqlQueries": [ { "accountId": 0, - "query": "SELECT average(elasticsearch.os.cpu.usage) as 'Avg OS CPU Used (%)' FROM Metric" + "query": "SELECT rate(sum(elasticsearch.cluster.state_update.time), 1 minute) / 1000 AS 'Cluster State Update Time (s/min)' FROM Metric TIMESERIES AUTO" } ] } }, { - "title": "Total JVM Heap Used (GB)", + "title": "Cluster Health Status", "layout": { "column": 9, - "row": 7, + "row": 5, "width": 4, "height": 3 }, "visualization": { - "id": "viz.billboard" + "id": "viz.line" }, "rawConfiguration": { + "colors": { + "seriesOverrides": [ + { + "color": "#FF0000", + "seriesName": "red" + }, + { + "color": "#008000", + "seriesName": "green" + }, + { + "color": "#FFFF00", + "seriesName": "yellow" + } + ] + }, "nrqlQueries": [ { "accountId": 0, - "query": "SELECT sum(jvm.memory.heap.used) / 1073741824 as 'Total JVM Heap Used (GB)' FROM Metric" + "query": "FROM Metric SELECT latest(elasticsearch.cluster.health) FACET status TIMESERIES AUTO" } ] } diff --git a/entity-types/infra-elasticsearchnode/definition.stg.yml b/entity-types/infra-elasticsearchnode/definition.stg.yml index 3780801ca..55dc7db8f 100644 --- a/entity-types/infra-elasticsearchnode/definition.stg.yml +++ b/entity-types/infra-elasticsearchnode/definition.stg.yml @@ -20,7 +20,12 @@ synthesis: - elasticsearch.node.name - elasticsearch.cluster.name - host.id - name: elasticsearch.node.name + compositeName: + fragments: + - value: "es-node=" + - attribute: elasticsearch.node.name + - value: ":es-cluster=" + - attribute: elasticsearch.cluster.name encodeIdentifierInGUID: true conditions: - attribute: eventType @@ -35,12 +40,49 @@ synthesis: present: true - attribute: elasticsearch.cluster.name present: true + - attribute: host.id + present: true tags: elasticsearch.node.name: elasticsearch.cluster.name: host.name: otel.library.name: entityTagName: instrumentation.name + - ruleName: infra_elasticsearchnode_composite_1 + compositeIdentifier: + separator: ":" + attributes: + - elasticsearch.node.name + - elasticsearch.cluster.name + - host.id + compositeName: + fragments: + - value: "es-node=" + - attribute: elasticsearch.node.name + - value: ":es-cluster=" + - attribute: elasticsearch.cluster.name + encodeIdentifierInGUID: true + conditions: + - attribute: eventType + value: Metric + - attribute: instrumentation.provider + value: opentelemetry + # This filters only metrics coming from elasticsearch receiver, given that metrics + # could differ between different runtime receivers. + - attribute: otel.library.name + value: "otelcol/elasticsearchreceiver" + - attribute: elasticsearch.node.name + present: true + - attribute: elasticsearch.cluster.name + present: true + - attribute: host.id + present: true + tags: + elasticsearch.node.name: + elasticsearch.cluster.name: + host.name: + otel.library.name: + entityTagName: instrumentation.name tags: # For OpenTelemetry telemetry.sdk.name: diff --git a/entity-types/infra-elasticsearchnode/golden_metrics.stg.yml b/entity-types/infra-elasticsearchnode/golden_metrics.stg.yml index 76d847bc3..1a7be0662 100644 --- a/entity-types/infra-elasticsearchnode/golden_metrics.stg.yml +++ b/entity-types/infra-elasticsearchnode/golden_metrics.stg.yml @@ -12,7 +12,7 @@ activeSearches: from: Metric where: operation='query' AND metricName = 'elasticsearch.node.operations.current' eventId: entity.guid - eventName: elasticsearch.node.name + eventName: entity.name missingDocumentRequests: title: Missing document requests unit: COUNT @@ -23,11 +23,11 @@ missingDocumentRequests: eventId: entityGuid eventName: entityName opentelemetry: - select: average(getField(elasticsearch.node.operations.get.completed, cumulative)) + select: average(elasticsearch.node.operations.get.completed) from: Metric where: result='miss' AND metricName = 'elasticsearch.node.operations.get.completed' eventId: entity.guid - eventName: elasticsearch.node.name + eventName: entity.name fileStoreIOOperations: title: File store I/O operations unit: COUNT @@ -37,7 +37,12 @@ fileStoreIOOperations: from: ElasticsearchNodeSample eventId: entityGuid eventName: entityName - # This metric is not available in the current OTEL Elasticsearch receiver + opentelemetry: + select: average(elasticsearch.node.disk.io.read) + from: Metric + where: metricName = 'elasticsearch.node.disk.io.read' + eventId: entity.guid + eventName: entity.name activeMerges: queries: newRelic: @@ -46,11 +51,11 @@ activeMerges: from: ElasticsearchNodeSample select: max(`merges.currentActive`) opentelemetry: - select: max(`elasticsearch.node.operations.current`) + select: max(elasticsearch.node.operations.current) from: Metric where: metricName = 'elasticsearch.node.operations.current' and operation = 'merge' eventId: entity.guid - eventName: elasticsearch.node.name + eventName: entity.name unit: COUNT title: Current active merges totalQueries: @@ -63,8 +68,8 @@ totalQueries: select: average(`queriesTotal`) from: ElasticsearchNodeSample opentelemetry: - select: average(getField(`elasticsearch.node.operations.completed`, cumulative)) + select: average(elasticsearch.node.operations.completed) from: Metric where: operation='query' AND metricName = 'elasticsearch.node.operations.completed' eventId: entity.guid - eventName: elasticsearch.node.name + eventName: entity.name diff --git a/entity-types/infra-elasticsearchnode/opentelemetry_dashboard.stg.json b/entity-types/infra-elasticsearchnode/opentelemetry_dashboard.stg.json index b6836aa16..1998bbc8f 100644 --- a/entity-types/infra-elasticsearchnode/opentelemetry_dashboard.stg.json +++ b/entity-types/infra-elasticsearchnode/opentelemetry_dashboard.stg.json @@ -15,7 +15,7 @@ "title" : "Running GET requests and Missing requests", "rawConfiguration" : { "nrqlQueries" : [ { - "query" : "SELECT filter(average(`elasticsearch.node.operations.current`), where operation='get' and metricName = 'elasticsearch.node.operations.current' ) AS `Running`, filter(average(getField(elasticsearch.node.operations.get.completed, cumulative)), where result='miss' and metricName = 'elasticsearch.node.operations.get.completed' ) AS 'Missing' FROM Metric TIMESERIES AUTO", + "query" : "SELECT filter(average(`elasticsearch.node.operations.current`), where operation='get' and metricName = 'elasticsearch.node.operations.current' ) AS `Running`, filter(average(`elasticsearch.node.operations.get.completed`), where result='miss' and metricName = 'elasticsearch.node.operations.get.completed' ) AS 'Missing' FROM Metric TIMESERIES AUTO", "accountId": 0} ] } }, { @@ -79,7 +79,7 @@ "title" : "Time spent indexing and deleting documents (secs)", "rawConfiguration" : { "nrqlQueries" : [ { - "query" : "SELECT average(getField(elasticsearch.node.operations.time, cumulative))/1000 FROM Metric WHERE metricName = 'elasticsearch.node.operations.time' facet operation in ('index', 'delete') TIMESERIES AUTO", + "query" : "SELECT average(elasticsearch.node.operations.time)/1000 FROM Metric WHERE metricName = 'elasticsearch.node.operations.time' facet operation in ('index', 'delete') TIMESERIES AUTO", "accountId": 0} ] } }, { diff --git a/entity-types/infra-kafkacluster/golden_metrics.stg.yml b/entity-types/infra-kafkacluster/golden_metrics.stg.yml index 8c88761e4..4c13e89a8 100644 --- a/entity-types/infra-kafkacluster/golden_metrics.stg.yml +++ b/entity-types/infra-kafkacluster/golden_metrics.stg.yml @@ -25,7 +25,7 @@ TopicCount: opentelemetry: select: latest(kafka.cluster.topic.count) from: Metric - where: metricName='kafka.cluster.topic.count' and kafka.cluster.topic.count > 0 + where: metricName='kafka.cluster.topic.count' and getField(kafka.cluster.topic.count , 'count') > 0 eventId: entity.guid partitionCount: @@ -40,7 +40,7 @@ partitionCount: opentelemetry: select: latest(kafka.cluster.partition.count) from: Metric - where: metricName='kafka.cluster.partition.count' and kafka.cluster.partition.count > 0 + where: metricName='kafka.cluster.partition.count' and getField(kafka.cluster.partition.count , 'count') > 0 eventId: entity.guid offlinePartitions: diff --git a/relationships/synthesis/APM-APPLICATION-to-INFRA-ELASTICSEARCHCLUSTER.stg.yml b/relationships/synthesis/EXT-SERVICE-to-INFRA-ELASTICSEARCHCLUSTER.stg.yml similarity index 100% rename from relationships/synthesis/APM-APPLICATION-to-INFRA-ELASTICSEARCHCLUSTER.stg.yml rename to relationships/synthesis/EXT-SERVICE-to-INFRA-ELASTICSEARCHCLUSTER.stg.yml diff --git a/relationships/synthesis/INFRA-DOCKER_CONTAINER-to-INFRA-ELASTICSEARCHNODE.stg.yml b/relationships/synthesis/INFRA-DOCKER_CONTAINER-to-INFRA-ELASTICSEARCHNODE.stg.yml new file mode 100644 index 000000000..e5935da8f --- /dev/null +++ b/relationships/synthesis/INFRA-DOCKER_CONTAINER-to-INFRA-ELASTICSEARCHNODE.stg.yml @@ -0,0 +1,36 @@ +relationships: + - name: containerHostsOtelElasticSearchNode + version: "1" + origins: + - OpenTelemetry + conditions: + - attribute: eventType + anyOf: [ "Metric" ] + - attribute: instrumentation.provider + anyOf: [ "opentelemetry" ] + - attribute: elasticsearch.node.name + present: true + - attribute: entity.type + anyOf: ["ELASTICSEARCHNODE"] + - attribute: container.id + present: true + relationship: + expires: PT75M + relationshipType: HOSTS + source: + buildGuid: + account: + attribute: accountId + domain: + value: INFRA + type: + value: CONTAINER + identifier: + fragments: + - attribute: container.id + hashAlgorithm: FARM_HASH + target: + extractGuid: + attribute: entity.guid + entityType: + attribute: entity.type \ No newline at end of file diff --git a/relationships/synthesis/INFRA-HOST-to-ELASTICSEARCHNODE.stg.yml b/relationships/synthesis/INFRA-HOST-to-ELASTICSEARCHNODE.stg.yml index 3d61dcac5..70b7aae83 100644 --- a/relationships/synthesis/INFRA-HOST-to-ELASTICSEARCHNODE.stg.yml +++ b/relationships/synthesis/INFRA-HOST-to-ELASTICSEARCHNODE.stg.yml @@ -6,12 +6,16 @@ relationships: conditions: - attribute: eventType anyOf: [ "Metric" ] + - attribute: instrumentation.provider + anyOf: [ "opentelemetry" ] - attribute: elasticsearch.node.name present: true - attribute: entity.type anyOf: ["ELASTICSEARCHNODE"] - attribute: host.id - present: true + present: true + - attribute: container.id + present: false relationship: expires: PT75M relationshipType: HOSTS diff --git a/relationships/synthesis/INFRA-HOST-to-INFRA-KAFKABROKER.stg.yml b/relationships/synthesis/INFRA-HOST-to-INFRA-KAFKABROKER.stg.yml index 1a5e40d33..e66d700b3 100644 --- a/relationships/synthesis/INFRA-HOST-to-INFRA-KAFKABROKER.stg.yml +++ b/relationships/synthesis/INFRA-HOST-to-INFRA-KAFKABROKER.stg.yml @@ -205,3 +205,31 @@ relationships: entityType: value: KAFKABROKER + # Host measures Kafka broker with OpenTelemetry when broker.endpoint is not present + - name: hostMonitorsOtelKafkaBroker2 + version: "1" + origins: + - OpenTelemetry + conditions: + - attribute: eventType + anyOf: [ "Metric" ] + - attribute: instrumentation.provider + anyOf: [ "opentelemetry" ] + - attribute: host.id + present: true + - attribute: broker.endpoint + present: false + relationship: + expires: PT75M + relationshipType: MEASURES + source: + lookupGuid: + candidateCategory: HOST + fields: + - field: hostId + attribute: host.id + target: + extractGuid: + attribute: entity.guid + entityType: + value: KAFKABROKER \ No newline at end of file diff --git a/relationships/synthesis/INFRA-KAFKABROKER-to-INFRA-KAFKATOPIC.stg.yml b/relationships/synthesis/INFRA-KAFKABROKER-to-INFRA-KAFKATOPIC.stg.yml index 12a6f0562..fcd37519b 100644 --- a/relationships/synthesis/INFRA-KAFKABROKER-to-INFRA-KAFKATOPIC.stg.yml +++ b/relationships/synthesis/INFRA-KAFKABROKER-to-INFRA-KAFKATOPIC.stg.yml @@ -82,6 +82,8 @@ relationships: anyOf: [ "KAFKATOPIC" ] - attribute: broker.id present: true + - attribute: kafka.cluster.name + present: true relationship: expires: PT75M relationshipType: MANAGES @@ -97,7 +99,7 @@ relationships: fragments: - attribute: broker.id - value: ":" - - attribute: cluster.name + - attribute: kafka.cluster.name hashAlgorithm: FARM_HASH target: extractGuid: