Skip to content

Commit

Permalink
feat(obsrvability-lib): refactor filters for Core Node General dashboard
Browse files Browse the repository at this point in the history
  • Loading branch information
@Atrax1
Atrax1 committed Jun 10, 2024
1 parent 316b5eb commit 189f664
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 49 deletions.
1 change: 1 addition & 0 deletions observability-lib/atlas-don/platform.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ func PlatformPanelOpts(platform string, ocrVersion string) PlatformOpts {
LabelFilters: map[string]string{
"contract": `=~"${contract}"`,
},
Platform: platform,
}

variableFeedID := "feed_id"
Expand Down
89 changes: 47 additions & 42 deletions observability-lib/core-node/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,29 @@ func BuildDashboard(name string, dataSourceMetric string, platform string) (dash

func vars(p Props) []cog.Builder[dashboard.VariableModel] {
var variables []cog.Builder[dashboard.VariableModel]
variables = append(variables,
utils.QueryVariable(p.MetricsDataSource, "instance", "Instance", fmt.Sprintf("label_values(%s)", p.PlatformOpts.LabelFilter), true))
variables = append(variables,
utils.QueryVariable(p.MetricsDataSource, "evmChainID", "EvmChainID", fmt.Sprintf("label_values(%s)", "evmChainID"), true))

//variables = append(variables,
// utils.QueryVariable(p.MetricsDataSource, "evmChainID", "EvmChainID", fmt.Sprintf("label_values(%s)", "evmChainID"), true))

if p.PlatformOpts.Platform == "kubernetes" {
variables = append(variables,
utils.QueryVariable(p.MetricsDataSource, "env", "Environment", `label_values(up, env)`, false))
variables = append(variables,
utils.QueryVariable(p.MetricsDataSource, "cluster", "Cluster", `label_values(up{env="$env"}, cluster)`, false))
variables = append(variables,
utils.QueryVariable(p.MetricsDataSource, "namespace", "Namespace", `label_values(up{env="$env", cluster="$cluster"}, namespace)`, false))
variables = append(variables,
utils.QueryVariable(p.MetricsDataSource, "blockchain", "Blockchain", `label_values(up{env="$env", cluster="$cluster", namespace="$namespace"}, blockchain)`, false))
variables = append(variables,
utils.QueryVariable(p.MetricsDataSource, "product", "Product", `label_values(up{env="$env", cluster="$cluster", namespace="$namespace", blockchain="$blockchain"}, product)`, false))
variables = append(variables,
utils.QueryVariable(p.MetricsDataSource, "network_type", "Network Type", `label_values(up{env="$env", cluster="$cluster", namespace="$namespace", blockchain="$blockchain", product="$product"}, network_type)`, false))
variables = append(variables,
utils.QueryVariable(p.MetricsDataSource, "job", "Job", `label_values(up{env="$env", cluster="$cluster", namespace="$namespace", blockchain="$blockchain", product="$product", network_type="$network_type"}, job)`, true))
} else if p.PlatformOpts.Platform == "docker" {
variables = append(variables,
utils.QueryVariable(p.MetricsDataSource, "instance", "Instance", fmt.Sprintf("label_values(%s)", p.PlatformOpts.LabelFilter), true))
}

return variables
}
Expand Down Expand Up @@ -264,7 +283,7 @@ func logPoller(p Props) []cog.Builder[dashboard.Panel] {
common.BigValueTextModeValueAndName,
common.VizOrientationHorizontal,
utils.PrometheusQuery{
Query: `count(count by (evmChainID) (log_poller_query_duration_sum{job=~"$instance"}))`,
Query: `count(count by (evmChainID) (log_poller_query_duration_sum{` + p.PlatformOpts.LabelQuery + `}))`,
Legend: "Goroutines",
},
))
Expand All @@ -279,11 +298,11 @@ func logPoller(p Props) []cog.Builder[dashboard.Panel] {
"reqps",
common.LegendPlacementBottom,
utils.PrometheusQuery{
Query: `avg by (query) (sum by (query, job) (rate(log_poller_query_duration_count{job=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval])))`,
Query: `avg by (query) (sum by (query, job) (rate(log_poller_query_duration_count{` + p.PlatformOpts.LabelQuery + `}[$__rate_interval])))`,
Legend: "{{query}} - {{job}}",
},
utils.PrometheusQuery{
Query: `avg (sum by(job) (rate(log_poller_query_duration_count{job=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval])))`,
Query: `avg (sum by(job) (rate(log_poller_query_duration_count{` + p.PlatformOpts.LabelQuery + `}[$__rate_interval])))`,
Legend: "Total",
},
))
Expand All @@ -298,7 +317,7 @@ func logPoller(p Props) []cog.Builder[dashboard.Panel] {
"reqps",
common.LegendPlacementBottom,
utils.PrometheusQuery{
Query: `avg by (type) (sum by (type, job) (rate(log_poller_query_duration_count{job=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval])))`,
Query: `avg by (type) (sum by (type, job) (rate(log_poller_query_duration_count{` + p.PlatformOpts.LabelQuery + `}[$__rate_interval])))`,
Legend: "{{query}} - {{job}}",
},
))
Expand All @@ -313,7 +332,7 @@ func logPoller(p Props) []cog.Builder[dashboard.Panel] {
"",
common.LegendPlacementBottom,
utils.PrometheusQuery{
Query: `avg by (query) (log_poller_query_dataset_size{job=~"$instance", evmChainID=~"$evmChainID"})`,
Query: `avg by (query) (log_poller_query_dataset_size{` + p.PlatformOpts.LabelQuery + `})`,
Legend: "{{query}} - {{job}}",
},
))
Expand All @@ -328,7 +347,7 @@ func logPoller(p Props) []cog.Builder[dashboard.Panel] {
"",
common.LegendPlacementBottom,
utils.PrometheusQuery{
Query: `max by (query) (log_poller_query_dataset_size{job=~"$instance", evmChainID=~"$evmChainID"})`,
Query: `max by (query) (log_poller_query_dataset_size{` + p.PlatformOpts.LabelQuery + `})`,
Legend: "{{query}} - {{job}}",
},
))
Expand All @@ -343,7 +362,7 @@ func logPoller(p Props) []cog.Builder[dashboard.Panel] {
"",
common.LegendPlacementBottom,
utils.PrometheusQuery{
Query: `max by (evmChainID) (log_poller_query_dataset_size{job=~"$instance", evmChainID=~"$evmChainID"})`,
Query: `max by (evmChainID) (log_poller_query_dataset_size{` + p.PlatformOpts.LabelQuery + `})`,
Legend: "{{evmChainID}}",
},
))
Expand All @@ -358,7 +377,7 @@ func logPoller(p Props) []cog.Builder[dashboard.Panel] {
"ms",
common.LegendPlacementBottom,
utils.PrometheusQuery{
Query: `histogram_quantile(0.5, sum(rate(log_poller_query_duration_bucket{job=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval])) by (le, query)) / 1e6`,
Query: `histogram_quantile(0.5, sum(rate(log_poller_query_duration_bucket{` + p.PlatformOpts.LabelQuery + `}[$__rate_interval])) by (le, query)) / 1e6`,
Legend: "{{query}}",
},
))
Expand All @@ -373,7 +392,7 @@ func logPoller(p Props) []cog.Builder[dashboard.Panel] {
"ms",
common.LegendPlacementBottom,
utils.PrometheusQuery{
Query: `histogram_quantile(0.9, sum(rate(log_poller_query_duration_bucket{job=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval])) by (le, query)) / 1e6`,
Query: `histogram_quantile(0.9, sum(rate(log_poller_query_duration_bucket{` + p.PlatformOpts.LabelQuery + `}[$__rate_interval])) by (le, query)) / 1e6`,
Legend: "{{query}}",
},
))
Expand All @@ -388,7 +407,7 @@ func logPoller(p Props) []cog.Builder[dashboard.Panel] {
"ms",
common.LegendPlacementBottom,
utils.PrometheusQuery{
Query: `histogram_quantile(0.99, sum(rate(log_poller_query_duration_bucket{job=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval])) by (le, query)) / 1e6`,
Query: `histogram_quantile(0.99, sum(rate(log_poller_query_duration_bucket{` + p.PlatformOpts.LabelQuery + `}[$__rate_interval])) by (le, query)) / 1e6`,
Legend: "{{query}}",
},
))
Expand All @@ -403,7 +422,7 @@ func logPoller(p Props) []cog.Builder[dashboard.Panel] {
"ms",
common.LegendPlacementBottom,
utils.PrometheusQuery{
Query: `histogram_quantile(0.99, sum(rate(log_poller_query_duration_bucket{job=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval])) by (le, evmChainID)) / 1e6`,
Query: `histogram_quantile(0.99, sum(rate(log_poller_query_duration_bucket{` + p.PlatformOpts.LabelQuery + `}[$__rate_interval])) by (le, evmChainID)) / 1e6`,
Legend: "{{query}}",
},
))
Expand All @@ -418,7 +437,7 @@ func logPoller(p Props) []cog.Builder[dashboard.Panel] {
"",
common.LegendPlacementBottom,
utils.PrometheusQuery{
Query: `avg by (evmChainID) (log_poller_logs_inserted{job=~"$instance", evmChainID=~"$evmChainID"})`,
Query: `avg by (evmChainID) (log_poller_logs_inserted{` + p.PlatformOpts.LabelQuery + `})`,
Legend: "{{evmChainID}}",
},
))
Expand All @@ -433,7 +452,7 @@ func logPoller(p Props) []cog.Builder[dashboard.Panel] {
"",
common.LegendPlacementBottom,
utils.PrometheusQuery{
Query: `avg by (evmChainID) (rate(log_poller_logs_inserted{job=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval]))`,
Query: `avg by (evmChainID) (rate(log_poller_logs_inserted{` + p.PlatformOpts.LabelQuery + `}[$__rate_interval]))`,
Legend: "{{evmChainID}}",
},
))
Expand All @@ -448,7 +467,7 @@ func logPoller(p Props) []cog.Builder[dashboard.Panel] {
"",
common.LegendPlacementBottom,
utils.PrometheusQuery{
Query: `avg by (evmChainID) (log_poller_blocks_inserted{job=~"$instance", evmChainID=~"$evmChainID"})`,
Query: `avg by (evmChainID) (log_poller_blocks_inserted{` + p.PlatformOpts.LabelQuery + `})`,
Legend: "{{evmChainID}}",
},
))
Expand All @@ -463,7 +482,7 @@ func logPoller(p Props) []cog.Builder[dashboard.Panel] {
"",
common.LegendPlacementBottom,
utils.PrometheusQuery{
Query: `avg by (evmChainID) (rate(log_poller_blocks_inserted{job=~"$instance", evmChainID=~"$evmChainID"}[$__rate_interval]))`,
Query: `avg by (evmChainID) (rate(log_poller_blocks_inserted{` + p.PlatformOpts.LabelQuery + `}[$__rate_interval]))`,
Legend: "{{evmChainID}}",
},
))
Expand Down Expand Up @@ -974,7 +993,7 @@ func evmPoolLifecycle(p Props) []cog.Builder[dashboard.Panel] {
"Block",
common.LegendPlacementBottom,
utils.PrometheusQuery{
Query: `evm_pool_rpc_node_highest_seen_block{` + p.PlatformOpts.LabelQuery + `evmChainID="${evmChainID}"}`,
Query: `evm_pool_rpc_node_highest_seen_block{` + p.PlatformOpts.LabelQuery + `}`,
Legend: `{{` + p.PlatformOpts.LegendString + `}}`,
},
))
Expand Down Expand Up @@ -1058,7 +1077,7 @@ func nodesRPC(p Props) []cog.Builder[dashboard.Panel] {
common.BigValueTextModeValueAndName,
common.VizOrientationHorizontal,
utils.PrometheusQuery{
Query: `sum(multi_node_states{` + p.PlatformOpts.LabelQuery + `chainId=~"$evmChainID", state="Alive"}) by (` + p.PlatformOpts.LegendString + `, chainId)`,
Query: `sum(multi_node_states{` + p.PlatformOpts.LabelQuery + `state="Alive"}) by (` + p.PlatformOpts.LegendString + `, chainId)`,
Legend: `{{` + p.PlatformOpts.LegendString + `}} - {{chainId}}`,
},
))
Expand All @@ -1076,7 +1095,7 @@ func nodesRPC(p Props) []cog.Builder[dashboard.Panel] {
common.BigValueTextModeValueAndName,
common.VizOrientationHorizontal,
utils.PrometheusQuery{
Query: `sum(multi_node_states{` + p.PlatformOpts.LabelQuery + `chainId=~"$evmChainID", state="Closed"}) by (` + p.PlatformOpts.LegendString + `, chainId)`,
Query: `sum(multi_node_states{` + p.PlatformOpts.LabelQuery + `state="Closed"}) by (` + p.PlatformOpts.LegendString + `, chainId)`,
Legend: `{{` + p.PlatformOpts.LegendString + `}} - {{chainId}}`,
},
))
Expand All @@ -1094,7 +1113,7 @@ func nodesRPC(p Props) []cog.Builder[dashboard.Panel] {
common.BigValueTextModeValueAndName,
common.VizOrientationHorizontal,
utils.PrometheusQuery{
Query: `sum(multi_node_states{` + p.PlatformOpts.LabelQuery + `chainId=~"$evmChainID", state="Dialed"}) by (` + p.PlatformOpts.LegendString + `, chainId)`,
Query: `sum(multi_node_states{` + p.PlatformOpts.LabelQuery + `state="Dialed"}) by (` + p.PlatformOpts.LegendString + `, chainId)`,
Legend: `{{` + p.PlatformOpts.LegendString + `}} - {{chainId}}`,
},
))
Expand All @@ -1112,7 +1131,7 @@ func nodesRPC(p Props) []cog.Builder[dashboard.Panel] {
common.BigValueTextModeValueAndName,
common.VizOrientationHorizontal,
utils.PrometheusQuery{
Query: `sum(multi_node_states{` + p.PlatformOpts.LabelQuery + `chainId=~"$evmChainID", state="InvalidChainID"}) by (` + p.PlatformOpts.LegendString + `, chainId)`,
Query: `sum(multi_node_states{` + p.PlatformOpts.LabelQuery + `state="InvalidChainID"}) by (` + p.PlatformOpts.LegendString + `, chainId)`,
Legend: `{{` + p.PlatformOpts.LegendString + `}} - {{chainId}}`,
},
))
Expand All @@ -1130,7 +1149,7 @@ func nodesRPC(p Props) []cog.Builder[dashboard.Panel] {
common.BigValueTextModeValueAndName,
common.VizOrientationHorizontal,
utils.PrometheusQuery{
Query: `sum(multi_node_states{` + p.PlatformOpts.LabelQuery + `chainId=~"$evmChainID", state="OutOfSync"}) by (` + p.PlatformOpts.LegendString + `, chainId)`,
Query: `sum(multi_node_states{` + p.PlatformOpts.LabelQuery + `state="OutOfSync"}) by (` + p.PlatformOpts.LegendString + `, chainId)`,
Legend: `{{` + p.PlatformOpts.LegendString + `}} - {{chainId}}`,
},
))
Expand All @@ -1148,7 +1167,7 @@ func nodesRPC(p Props) []cog.Builder[dashboard.Panel] {
common.BigValueTextModeValueAndName,
common.VizOrientationHorizontal,
utils.PrometheusQuery{
Query: `sum(multi_node_states{` + p.PlatformOpts.LabelQuery + `chainId=~"$evmChainID", state="Undialed"}) by (` + p.PlatformOpts.LegendString + `, chainId)`,
Query: `sum(multi_node_states{` + p.PlatformOpts.LabelQuery + `state="Undialed"}) by (` + p.PlatformOpts.LegendString + `, chainId)`,
Legend: `{{` + p.PlatformOpts.LegendString + `}} - {{chainId}}`,
},
))
Expand All @@ -1166,7 +1185,7 @@ func nodesRPC(p Props) []cog.Builder[dashboard.Panel] {
common.BigValueTextModeValueAndName,
common.VizOrientationHorizontal,
utils.PrometheusQuery{
Query: `sum(multi_node_states{` + p.PlatformOpts.LabelQuery + `chainId=~"$evmChainID", state="Unreachable"}) by (` + p.PlatformOpts.LegendString + `, chainId)`,
Query: `sum(multi_node_states{` + p.PlatformOpts.LabelQuery + `state="Unreachable"}) by (` + p.PlatformOpts.LegendString + `, chainId)`,
Legend: `{{` + p.PlatformOpts.LegendString + `}} - {{chainId}}`,
},
))
Expand All @@ -1184,7 +1203,7 @@ func nodesRPC(p Props) []cog.Builder[dashboard.Panel] {
common.BigValueTextModeValueAndName,
common.VizOrientationHorizontal,
utils.PrometheusQuery{
Query: `sum(multi_node_states{` + p.PlatformOpts.LabelQuery + `chainId=~"$evmChainID", state="Unusable"}) by (` + p.PlatformOpts.LegendString + `, chainId)`,
Query: `sum(multi_node_states{` + p.PlatformOpts.LabelQuery + `state="Unusable"}) by (` + p.PlatformOpts.LegendString + `, chainId)`,
Legend: `{{` + p.PlatformOpts.LegendString + `}} - {{chainId}}`,
},
))
Expand Down Expand Up @@ -1606,20 +1625,6 @@ func httpAPI(p Props) []cog.Builder[dashboard.Panel] {
func promHTTP(p Props) []cog.Builder[dashboard.Panel] {
var panelsArray []cog.Builder[dashboard.Panel]

panelsArray = append(panelsArray, utils.GaugePanel(
p.MetricsDataSource,
"HTTP Request in flight",
"",
6,
24,
1,
"",
utils.PrometheusQuery{
Query: `promhttp_metric_handler_requests_in_flight{` + p.PlatformOpts.LabelQuery + `}`,
Legend: `{{` + p.PlatformOpts.LegendString + `}}`,
},
))

panelsArray = append(panelsArray, utils.TimeSeriesPanel(
p.MetricsDataSource,
"HTTP rate by return code",
Expand Down
11 changes: 4 additions & 7 deletions observability-lib/core-node/platform.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,19 +20,16 @@ type Props struct {
func PlatformPanelOpts(platform string) PlatformOpts {
po := PlatformOpts{
LabelFilters: map[string]string{},
Platform: platform,
}
switch platform {
case "kubernetes":
po.LabelFilters = map[string]string{
"namespace": `=~"${namespace}"`,
"pod": `=~"${pod}"`,
}
po.LabelFilters["namespace"] = `=~"${namespace}"`
po.LabelFilters["job"] = `=~"${job}"`
po.LabelFilter = "job"
po.LegendString = "pod"
case "docker":
po.LabelFilters = map[string]string{
"instance": `=~"${instance}"`,
}
po.LabelFilters["instance"] = `=~"${instance}"`
po.LabelFilter = "instance"
po.LegendString = "instance"
default:
Expand Down

0 comments on commit 189f664

Please sign in to comment.