Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature: add metrics about metasrv network and docs #5842

Merged
merged 2 commits into from
Jun 8, 2022
Merged

Feature: add metrics about metasrv network and docs #5842

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2b3aa2b
Feature: add metrics about metasrv network and docs
lichuang Jun 8, 2022
0e0dcdc
Merge branch 'main' into meta_network_metrics
mergify[bot] Jun 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 21 additions & 26 deletions common/meta/types/proto/meta.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
// Copyright 2021 Datafuse Labs.
//
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//
// http://www.apache.org/licenses/LICENSE-2.0
//
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Expand All @@ -26,22 +26,16 @@ message GetReply {
string value = 2;
}

message RaftRequest {
string data = 1;
}
message RaftRequest { string data = 1; }

message RaftReply {
string data = 1;
string error = 2;
}

message MemberListRequest {
string data = 1;
}
message MemberListRequest { string data = 1; }

message MemberListReply {
repeated string data = 1;
}
message MemberListReply { repeated string data = 1; }

message HandshakeRequest {
uint64 protocol_version = 1;
Expand All @@ -54,9 +48,7 @@ message HandshakeResponse {
}

// Data chunk for export/import meta data
message ExportedChunk {
repeated string data = 10;
}
message ExportedChunk { repeated string data = 10; }

message WatchRequest {
// key is the key to register for watching.
Expand All @@ -68,12 +60,12 @@ message WatchRequest {

enum FilterType {
// recv all kind update event.
ALL = 0;
ALL = 0;
// filter only update event.
UPDATE = 1;
// filter only delete event.
DELETE = 2;
}
}
FilterType filter_type = 3;
}

Expand All @@ -87,9 +79,7 @@ message Event {
optional SeqV prev = 3;
}

message WatchResponse {
Event event = 1;
}
message WatchResponse { Event event = 1; }

// messages for txn
message TxnCondition {
Expand All @@ -113,7 +103,8 @@ message TxnCondition {
}

// the expected result of condition, if `expected` match the condition result,
// then the `if_then` op will be executed, otherwise `else_then` op will be executed.
// then the `if_then` op will be executed, otherwise `else_then` op will be
// executed.
ConditionResult expected = 4;
}

Expand All @@ -135,14 +126,16 @@ message TxnOpResponse {

message TxnRequest {
// `condition` is a list of predicates.
// If all of them success, the `if_then` will be executed,
// If all of them success, the `if_then` will be executed,
// otherwise `else_then` op will be executed.
repeated TxnCondition condition = 1;

// `if_then` is a list of operations will be executed when all condition evaluates to true.
// `if_then` is a list of operations will be executed when all condition
// evaluates to true.
repeated TxnOp if_then = 2;

// `else_then` is a list of operations will be executed when not all condition evaluates to true.
// `else_then` is a list of operations will be executed when not all condition
// evaluates to true.
repeated TxnOp else_then = 3;
}

Expand Down Expand Up @@ -178,11 +171,13 @@ service MetaService {
// Export all meta data.
//
// Including raft hard state, logs and state machine.
// The exported data is a list of json strings in form of `(tree_name, sub_tree_prefix, key, value)`.
// The exported data is a list of json strings in form of `(tree_name,
// sub_tree_prefix, key, value)`.
rpc Export(Empty) returns (stream ExportedChunk);

// Add watch key stream.
// Whenever the watch key data updated, client will be notified accross the stream.
// Whenever the watch key data updated, client will be notified accross the
// stream.
rpc Watch(WatchRequest) returns (stream WatchResponse);

rpc Transaction(TxnRequest) returns (TxnReply);
Expand Down
36 changes: 34 additions & 2 deletions docs/doc/50-manage/00-metasrv/50-metasrv-metrics.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ These metrics describe the status of the `metasrv`. All these metrics are prefix

| Name | Description | Type |
| ----------------- | ------------------------------------------------- | ------- |
| has_leader | Whether or not a leader exists. | Gauge |
| current_leader_id | Current leader id of cluster, 0 means no leader. | IntGauge |
| is_leader | Whether or not this node is current leader. | Gauge |
| leader_changes | Number of leader changes seen. | Counter |
| applying_snapshot | Whether or not statemachine is applying snapshot. | Gauge |
Expand All @@ -28,7 +28,7 @@ These metrics describe the status of the `metasrv`. All these metrics are prefix
| proposals_failed | Total number of failed proposals. | Counter |
| watchers | Total number of active watchers. | Gauge |

`has_leader` indicate if there is a leader in the cluster, if a member in the cluster has no leader, it is unavailable.
`current_leader_id` indicate current leader id of cluster, 0 means no leader. If a cluster has no leader, it is unavailable.

`is_leader` indicate if this `metasrv` currently is the leader of cluster, and `leader_changes` show the total number of leader changes since start.If change leader too frequently, it will impact the performance of `metasrv`, also it signal that the cluster is unstable.

Expand All @@ -39,3 +39,35 @@ These metrics describe the status of the `metasrv`. All these metrics are prefix
`proposals_failed` show the total number of failed write requests, it is normally related to two issues: temporary failures related to a leader election or longer downtime caused by a loss of quorum in the cluster.

`watchers` show the total number of active watchers currently.

### Network

These metrics describe the network status of the `metasrv`. All these metrics are prefixed with `metasrv_network_`.

| Name | Description | Labels | Type |
| ----------------------- | ------------------------------------------------- | --------------------------------- | ------------- |
| active_peers | Current number of active connections to peers. | id(node id),address(peer address) | GaugeVec |
| fail_connect_to_peer | Total number of fail connections to peers. | id(node id),address(peer address) | CounterVec |
| sent_bytes | Total number of sent bytes to peers. | to(node id) | CounterVec |
| recv_bytes | Total number of received bytes from peers. | from(remote address) | CounterVec |
| sent_failures | Total number of send failures to peers. | to(node id) | CounterVec |
| snapshot_send_success | Total number of successful snapshot sends. | to(node id) | IntCounterVec |
| snapshot_send_failures | Total number of snapshot send failures. | to(node id) | IntCounterVec |
| snapshot_send_inflights | Total number of inflight snapshot sends. | to(node id) | IntGaugeVec |
| snapshot_sent_seconds | Total latency distributions of snapshot sends. | to(node id) | HistogramVec |
| snapshot_recv_success | Total number of successful receive snapshot. | from(remote address) | IntCounterVec |
| snapshot_recv_failures | Total number of snapshot receive failures. | from(remote address) | IntCounterVec |
| snapshot_recv_inflights | Total number of inflight snapshot receives. | from(remote address) | IntGaugeVec |
| snapshot_recv_seconds | Total latency distributions of snapshot receives. | from(remote address) | HistogramVec |

`active_peers` indicates how many active connection between cluster members, `fail_connect_to_peer` indicates the number of fail connections to peers. Each has the labels: id(node id) and address (peer address).

`sent_bytes` and `recv_bytes` record the sent and receive bytes to and from peers, and `sent_failures` records the number of fail sent to peers.

`snapshot_send_success` and `snapshot_send_failures` indicates the success and fail number of sent snapshot.`snapshot_send_inflights` indicate the inflight snapshot sends, each time send a snapshot, this field will increment by one, after sending snapshot is done, this field will decrement by one.

`snapshot_sent_seconds` indicate the total latency distributions of snapshot sends.

`snapshot_recv_success` and `snapshot_recv_failures` indicates the success and fail number of receive snapshot.`snapshot_recv_inflights` indicate the inflight receiving snapshot, each time receive a snapshot, this field will increment by one, after receiving snapshot is done, this field will decrement by one.

`snapshot_recv_seconds` indicate the total latency distributions of snapshot receives.
51 changes: 44 additions & 7 deletions metasrv/src/meta_service/meta_service_impl.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
use std::convert::TryInto;
use std::pin::Pin;
use std::sync::Arc;
use std::time::Instant;

use common_meta_types::protobuf::raft_service_server::RaftService;
use common_meta_types::protobuf::GetReply;
Expand All @@ -36,6 +37,11 @@ use crate::meta_service::ForwardRequestBody;
use crate::meta_service::MetaNode;
use crate::metrics::incr_meta_metrics_proposals_failed;
use crate::metrics::incr_meta_metrics_proposals_pending;
use crate::metrics::incr_meta_metrics_recv_bytes_from_peer;
use crate::metrics::incr_meta_metrics_snapshot_recv_failure_from_peer;
use crate::metrics::incr_meta_metrics_snapshot_recv_inflights_from_peer;
use crate::metrics::incr_meta_metrics_snapshot_recv_success_from_peer;
use crate::metrics::sample_meta_metrics_snapshot_recv;

pub type GrpcStream<T> =
Pin<Box<dyn Stream<Item = Result<T, tonic::Status>> + Send + Sync + 'static>>;
Expand All @@ -48,6 +54,14 @@ impl RaftServiceImpl {
pub fn create(meta_node: Arc<MetaNode>) -> Self {
Self { meta_node }
}

fn incr_meta_metrics_recv_bytes_from_peer(&self, request: &tonic::Request<RaftRequest>) {
if let Some(addr) = request.remote_addr() {
let message: &RaftRequest = request.get_ref();
let bytes = message.data.len() as u64;
incr_meta_metrics_recv_bytes_from_peer(addr.to_string(), bytes);
}
}
}

#[async_trait::async_trait]
Expand Down Expand Up @@ -138,6 +152,7 @@ impl RaftService for RaftServiceImpl {
) -> Result<tonic::Response<RaftReply>, tonic::Status> {
common_tracing::extract_remote_span_as_parent(&request);

self.incr_meta_metrics_recv_bytes_from_peer(&request);
let req = request.into_inner();

let ae_req =
Expand All @@ -163,8 +178,16 @@ impl RaftService for RaftServiceImpl {
&self,
request: tonic::Request<RaftRequest>,
) -> Result<tonic::Response<RaftReply>, tonic::Status> {
let start = Instant::now();
let addr = if let Some(addr) = request.remote_addr() {
addr.to_string()
} else {
"unknown address".to_string()
};
common_tracing::extract_remote_span_as_parent(&request);

self.incr_meta_metrics_recv_bytes_from_peer(&request);
incr_meta_metrics_snapshot_recv_inflights_from_peer(addr.clone(), 1);
let req = request.into_inner();

let is_req =
Expand All @@ -175,14 +198,27 @@ impl RaftService for RaftServiceImpl {
.raft
.install_snapshot(is_req)
.await
.map_err(|x| tonic::Status::internal(x.to_string()))?;
let data = serde_json::to_string(&resp).expect("fail to serialize resp");
let mes = RaftReply {
data,
error: "".to_string(),
};
.map_err(|x| tonic::Status::internal(x.to_string()));

Ok(tonic::Response::new(mes))
sample_meta_metrics_snapshot_recv(addr.clone(), start.elapsed().as_secs() as f64);
incr_meta_metrics_snapshot_recv_inflights_from_peer(addr.clone(), -1);

match resp {
Ok(resp) => {
let data = serde_json::to_string(&resp).expect("fail to serialize resp");
let mes = RaftReply {
data,
error: "".to_string(),
};

incr_meta_metrics_snapshot_recv_success_from_peer(addr.clone());
return Ok(tonic::Response::new(mes));
}
Err(e) => {
incr_meta_metrics_snapshot_recv_failure_from_peer(addr.clone());
return Err(e);
}
}
}

#[tracing::instrument(level = "debug", skip(self, request))]
Expand All @@ -192,6 +228,7 @@ impl RaftService for RaftServiceImpl {
) -> Result<tonic::Response<RaftReply>, tonic::Status> {
common_tracing::extract_remote_span_as_parent(&request);

self.incr_meta_metrics_recv_bytes_from_peer(&request);
let req = request.into_inner();

let v_req =
Expand Down
6 changes: 3 additions & 3 deletions metasrv/src/meta_service/raftmeta.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ use crate::meta_service::JoinRequest;
use crate::meta_service::RaftServiceImpl;
use crate::metrics::incr_meta_metrics_leader_change;
use crate::metrics::incr_meta_metrics_read_failed;
use crate::metrics::set_meta_metrics_has_leader;
use crate::metrics::set_meta_metrics_current_leader;
use crate::metrics::set_meta_metrics_is_leader;
use crate::metrics::set_meta_metrics_proposals_applied;
use crate::network::Network;
Expand Down Expand Up @@ -405,9 +405,9 @@ impl MetaNode {
} else {
set_meta_metrics_is_leader(false);
}
set_meta_metrics_has_leader(true);
set_meta_metrics_current_leader(cur);
} else {
set_meta_metrics_has_leader(false);
set_meta_metrics_current_leader(0);
set_meta_metrics_is_leader(false);
}
if let Some(last_applied) = mm.last_applied {
Expand Down
Loading