diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 000000000..f9936e3af --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,11 @@ +coverage: + status: + project: #add everything under here, more options at https://docs.codecov.com/docs/commit-status + default: + # basic + target: auto #default + threshold: 50% + base: auto +comment: # this is a top-level key + layout: "header, files, footer" # remove "new" from "header" and "footer" + hide_project_coverage: false # set to false \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/enhancement.md b/.github/ISSUE_TEMPLATE/enhancement.md index 27eca10a1..c64b88cfa 100644 --- a/.github/ISSUE_TEMPLATE/enhancement.md +++ b/.github/ISSUE_TEMPLATE/enhancement.md @@ -1,7 +1,7 @@ --- name: Enhancement Request about: Suggest an enhancement to the project -labels: kind/feature +labels: kind/enhancement --- diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1a54872e3..146695143 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,6 +3,10 @@ name: Kmesh CI Workflow on: pull_request: merge_group: # enable merge queue + workflow_call: + secrets: + CODECOV_TOKEN: + required: true jobs: @@ -24,6 +28,10 @@ jobs: run: | make gen-check + - name: Run Copyright-check + run : | + make ./hack/copyright-check.sh + - name: Build Kmesh shell: bash run: | @@ -45,9 +53,13 @@ jobs: sudo env LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:$GITHUB_WORKSPACE/api/v2-c:$GITHUB_WORKSPACE/bpf/deserialization_to_bpf_map PKG_CONFIG_PATH=$GITHUB_WORKSPACE/mk go test -race -v -vet=off -coverprofile=coverage.out ./pkg/... - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4.0.1 + uses: codecov/codecov-action@v4 with: - token: ${{ secrets.CODECOV_TOKEN }} - files: coverage.out - flags: unittests - fail_ci_if_error: false \ No newline at end of file + # Even though token upload token is not required for public repos, + # but adding a token might increase successful uploads as per: + # https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954 + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.out + flags: unittests + fail_ci_if_error: false + verbose: true diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..d0ea277d1 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1 @@ +For further details please see [Security Policy](https://github.com/kmesh-net/community/blob/main/security-team/SECURITY.md) for our security process and how to report vulnerabilities. \ No newline at end of file diff --git a/bpf/include/bpf_common.h b/bpf/include/bpf_common.h index 9b8299a4a..7691472d1 100644 --- a/bpf/include/bpf_common.h +++ b/bpf/include/bpf_common.h @@ -46,9 +46,22 @@ struct { __type(key, struct manager_key); __type(value, manager_value_t); __uint(max_entries, MAP_SIZE_OF_MANAGER); - __uint(map_flags, 0); + __uint(map_flags, BPF_F_NO_PREALLOC); } map_of_manager SEC(".maps"); +struct sock_storage_data { + __u64 connect_ns; + __u8 direction; + __u8 connect_success; +}; + +struct { + __uint(type, BPF_MAP_TYPE_SK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct sock_storage_data); +} map_of_sock_storage SEC(".maps"); + /* * From v5.4, bpf_get_netns_cookie can be called for bpf cgroup hooks, from v5.15, it can be called for bpf sockops * hook. Therefore, ensure that function is correctly used. @@ -81,7 +94,7 @@ static inline void set_netns_bypass_value(struct bpf_sock_addr *ctx, int new_byp current_value->is_bypassed = new_bypass_value; - int err = bpf_map_update_elem(&map_of_manager, &key, current_value, BPF_EXIST); + int err = bpf_map_update_elem(&map_of_manager, &key, current_value, BPF_ANY); if (err) BPF_LOG(ERR, KMESH, "set netcookie failed!, err is %d\n", err); } diff --git a/bpf/include/bpf_log.h b/bpf/include/bpf_log.h index a91906ce1..0eb37f9f6 100644 --- a/bpf/include/bpf_log.h +++ b/bpf/include/bpf_log.h @@ -16,6 +16,7 @@ #define BPF_LOGTYPE_SOCKOPS BPF_DEBUG_OFF #define BPF_LOGTYPE_XDP BPF_DEBUG_OFF #define BPF_LOGTYPE_SENDMSG BPF_DEBUG_OFF +#define BPF_LOGTYPE_PROBE BPF_DEBUG_ON #define MAX_MSG_LEN 255 enum bpf_loglevel { diff --git a/bpf/kmesh/ads/include/cluster.h b/bpf/kmesh/ads/include/cluster.h index 0312eeb8a..4dcd52cc4 100644 --- a/bpf/kmesh/ads/include/cluster.h +++ b/bpf/kmesh/ads/include/cluster.h @@ -47,7 +47,7 @@ struct { __uint(max_entries, 1); } map_of_cluster_eps_data SEC(".maps"); -static inline Cluster__Cluster *map_lookup_cluster_eps_data() +static inline struct cluster_endpoints *map_lookup_cluster_eps_data() { int location = 0; return kmesh_map_lookup_elem(&map_of_cluster_eps_data, &location); diff --git a/bpf/kmesh/ads/include/kmesh_common.h b/bpf/kmesh/ads/include/kmesh_common.h index deb4ad85a..abedaad44 100644 --- a/bpf/kmesh/ads/include/kmesh_common.h +++ b/bpf/kmesh/ads/include/kmesh_common.h @@ -37,7 +37,7 @@ struct bpf_mem_ptr { }; #if !ENHANCED_KERNEL -static inline int bpf__strncmp(char *dst, int n, const char *src) +static inline int bpf__strncmp(const char *dst, int n, const char *src) { if (dst == NULL || src == NULL) return -1; diff --git a/bpf/kmesh/bpf2go/bpf2go.go b/bpf/kmesh/bpf2go/bpf2go.go index 5eeee960c..cc6182a9e 100644 --- a/bpf/kmesh/bpf2go/bpf2go.go +++ b/bpf/kmesh/bpf2go/bpf2go.go @@ -1,5 +1,18 @@ -// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) -/* Copyright Authors of Kmesh */ +/* + * Copyright The Kmesh Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ //go:build tools // +build tools @@ -13,9 +26,9 @@ import ( // go run github.com/cilium/ebpf/cmd/bpf2go --help //go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshCgroupSock ../ads/cgroup_sock.c -- -I../ads/include -I../../include -I../../../api/v2-c -DCGROUP_SOCK_MANAGE -//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshCgroupSockWorkload ../workload/cgroup_sock.c -- -I../workload/include -I../../include +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshCgroupSockWorkload ../workload/cgroup_sock.c -- -I../workload/include -I../../include -I../probes //go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshSockops ../ads/sockops.c -- -I../ads/include -I../../include -I../../../api/v2-c //go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshTracePoint ../ads/tracepoint.c -- -I../ads/include -I../../include -//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshSockopsWorkload ../workload/sockops.c -- -I../workload/include -I../../include +//go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshSockopsWorkload ../workload/sockops.c -- -I../workload/include -I../../include -I../probes //go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshXDPAuth ../workload/xdp.c -- -I../workload/include -I../../include //go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cc clang --cflags $EXTRA_CFLAGS --cflags $EXTRA_CDEFINE KmeshSendmsg ../workload/sendmsg.c -- -I../workload/include -I../../include diff --git a/bpf/kmesh/probes/access_log.h b/bpf/kmesh/probes/access_log.h new file mode 100644 index 000000000..618b4b378 --- /dev/null +++ b/bpf/kmesh/probes/access_log.h @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright Authors of Kmesh */ + +#ifndef __KMESH_BPF_ACCESS_LOG_H__ +#define __KMESH_BPF_ACCESS_LOG_H__ + +#include "bpf_common.h" + +// access log +enum { + INVALID_DIRECTION = 0, + INBOUND = 1, + OUTBOUND = 2, +}; + +struct access_log { + struct bpf_sock_tuple tuple; + __u64 duration; // ns + __u64 close_ns; + __u32 family; + __u32 protocol; + __u8 direction; + __u32 sent_bytes; + __u32 received_bytes; + __u32 conn_success; +}; + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, RINGBUF_SIZE); +} map_of_access_log SEC(".maps"); + +static inline void constuct_tuple(struct bpf_sock *sk, struct bpf_sock_tuple *tuple) +{ + if (sk->family == AF_INET) { + tuple->ipv4.saddr = sk->src_ip4; + tuple->ipv4.daddr = sk->dst_ip4; + tuple->ipv4.sport = sk->src_port; + tuple->ipv4.dport = sk->dst_port; + } else { + bpf_memcpy(tuple->ipv6.saddr, sk->src_ip6, IPV6_ADDR_LEN); + bpf_memcpy(tuple->ipv6.daddr, sk->dst_ip6, IPV6_ADDR_LEN); + tuple->ipv6.sport = sk->src_port; + tuple->ipv6.dport = sk->dst_port; + } + return; +} + +static inline void +report_access_log(struct bpf_sock *sk, struct bpf_tcp_sock *tcp_sock, struct sock_storage_data *storage) +{ + struct access_log *log = NULL; + + // store tuple + log = bpf_ringbuf_reserve(&map_of_access_log, sizeof(struct access_log), 0); + if (!log) { + BPF_LOG(ERR, PROBE, "bpf_ringbuf_reserve access_log failed\n"); + return; + } + + constuct_tuple(sk, &log->tuple); + log->direction = storage->direction; + log->close_ns = bpf_ktime_get_ns(); + log->duration = log->close_ns - storage->connect_ns; + log->sent_bytes = tcp_sock->delivered; + log->received_bytes = tcp_sock->bytes_received; + log->conn_success = storage->connect_success; + + bpf_ringbuf_submit(log, 0); +} + +#endif \ No newline at end of file diff --git a/bpf/kmesh/probes/metrics.h b/bpf/kmesh/probes/metrics.h new file mode 100644 index 000000000..3db66d29b --- /dev/null +++ b/bpf/kmesh/probes/metrics.h @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright Authors of Kmesh */ + +#ifndef __KMESH_BPF_METRICS_H__ +#define __KMESH_BPF_METRICS_H__ +#include "bpf_common.h" + +// metrics +struct metric_key { + struct ip_addr src_ip; + struct ip_addr dst_ip; +}; + +struct metric_data { + __u32 direction; // update on connect + __u32 conn_open; // update on connect + __u32 conn_close; // update on close + __u32 conn_failed; // update on close + __u32 sent_bytes; // update on close + __u32 received_bytes; // update on close +}; + +#define MAP_SIZE_OF_METRICS 100000 +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, struct metric_key); + __type(value, struct metric_data); + __uint(max_entries, MAP_SIZE_OF_METRICS); + __uint(map_flags, BPF_F_NO_PREALLOC); +} map_of_metrics SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_RINGBUF); + __uint(max_entries, RINGBUF_SIZE); +} map_of_metric_notify SEC(".maps"); + +static inline void construct_metric_key(struct bpf_sock *sk, struct metric_key *key) +{ + if (sk->family == AF_INET) { + key->src_ip.ip4 = sk->src_ip4; + key->dst_ip.ip4 = sk->dst_ip4; + } else { + bpf_memcpy(key->src_ip.ip6, sk->src_ip6, IPV6_ADDR_LEN); + bpf_memcpy(key->dst_ip.ip6, sk->dst_ip6, IPV6_ADDR_LEN); + } + return; +} + +static inline void report_metrics(struct bpf_sock *sk) +{ + struct metric_key *key = bpf_ringbuf_reserve(&map_of_metric_notify, sizeof(struct metric_key), 0); + if (!key) { + BPF_LOG(ERR, PROBE, "report_metrics bpf_ringbuf_reserve failed\n"); + return; + } + + construct_metric_key(sk, key); + bpf_ringbuf_submit(key, 0); + return; +} + +static inline void +metric_on_connect(struct bpf_sock *sk, struct bpf_tcp_sock *tcp_sock, struct sock_storage_data *storage) +{ + struct metric_key key = {0}; + struct metric_data data = {0}; + struct metric_data *metric = NULL; + + construct_metric_key(sk, &key); + metric = (struct metric_data *)bpf_map_lookup_elem(&map_of_metrics, &key); + if (!metric) { + data.conn_open++; + data.direction = storage->direction; + int err = bpf_map_update_elem(&map_of_metrics, &key, &data, BPF_NOEXIST); + if (err) { + BPF_LOG(ERR, PROBE, "metric_on_connect update failed, err is %d\n", err); + return; + } + goto notify; + } + + metric->conn_open++; + metric->direction = storage->direction; +notify: + report_metrics(sk); + return; +} + +static inline void +metric_on_close(struct bpf_sock *sk, struct bpf_tcp_sock *tcp_sock, struct sock_storage_data *storage) +{ + struct metric_key key = {0}; + struct metric_data data = {0}; + struct metric_data *metric = NULL; + + construct_metric_key(sk, &key); + metric = (struct metric_data *)bpf_map_lookup_elem(&map_of_metrics, &key); + if (!metric) { + // connect failed + data.direction = storage->direction; + data.conn_failed++; + int err = bpf_map_update_elem(&map_of_metrics, &key, &data, BPF_NOEXIST); + if (err) { + BPF_LOG(ERR, PROBE, "metric_on_close update failed, err is %d\n", err); + return; + } + goto notify; + } + + // connect successed & closed + metric->conn_close++; + metric->sent_bytes += tcp_sock->delivered; + metric->received_bytes += tcp_sock->bytes_received; +notify: + report_metrics(sk); + return; +} + +#endif \ No newline at end of file diff --git a/bpf/kmesh/probes/probe.h b/bpf/kmesh/probes/probe.h new file mode 100644 index 000000000..d726b02e0 --- /dev/null +++ b/bpf/kmesh/probes/probe.h @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright Authors of Kmesh */ + +#ifndef __KMESH_BPF_PROBE_H__ +#define __KMESH_BPF_PROBE_H__ + +#include "access_log.h" +#include "metrics.h" + +static inline void observe_on_pre_connect(struct bpf_sock *sk) +{ + struct sock_storage_data *storage = NULL; + if (!sk) + return; + + storage = bpf_sk_storage_get(&map_of_sock_storage, sk, 0, BPF_LOCAL_STORAGE_GET_F_CREATE); + if (!storage) { + BPF_LOG(ERR, PROBE, "pre_connect bpf_sk_storage_get failed\n"); + return; + } + + storage->connect_ns = bpf_ktime_get_ns(); + return; +} + +static inline void observe_on_connect_established(struct bpf_sock *sk, __u8 direction) +{ + struct bpf_tcp_sock *tcp_sock = NULL; + struct sock_storage_data *storage = NULL; + __u64 flags = (direction == OUTBOUND) ? 0 : BPF_LOCAL_STORAGE_GET_F_CREATE; + + if (!sk) + return; + tcp_sock = bpf_tcp_sock(sk); + if (!tcp_sock) + return; + + storage = bpf_sk_storage_get(&map_of_sock_storage, sk, 0, flags); + if (!storage) { + BPF_LOG(ERR, PROBE, "connect bpf_sk_storage_get failed\n"); + return; + } + + // INBOUND senario + if (direction == INBOUND) + storage->connect_ns = bpf_ktime_get_ns(); + storage->direction = direction; + storage->connect_success = true; + + metric_on_connect(sk, tcp_sock, storage); +} + +static inline void observe_on_close(struct bpf_sock *sk) +{ + struct bpf_tcp_sock *tcp_sock = NULL; + struct sock_storage_data *storage = NULL; + if (!sk) + return; + tcp_sock = bpf_tcp_sock(sk); + if (!tcp_sock) + return; + + storage = bpf_sk_storage_get(&map_of_sock_storage, sk, 0, 0); + if (!storage) { + BPF_LOG(ERR, PROBE, "close bpf_sk_storage_get failed\n"); + return; + } + + report_access_log(sk, tcp_sock, storage); + metric_on_close(sk, tcp_sock, storage); +} +#endif \ No newline at end of file diff --git a/bpf/kmesh/workload/cgroup_sock.c b/bpf/kmesh/workload/cgroup_sock.c index 501370652..d3db5efe7 100644 --- a/bpf/kmesh/workload/cgroup_sock.c +++ b/bpf/kmesh/workload/cgroup_sock.c @@ -9,6 +9,7 @@ #include "ctx/sock_addr.h" #include "frontend.h" #include "bpf_common.h" +#include "probe.h" static inline int sock_traffic_control(struct kmesh_context *kmesh_ctx) { @@ -26,7 +27,7 @@ static inline int sock_traffic_control(struct kmesh_context *kmesh_ctx) KMESH, "origin addr=[%u:%s:%u]\n", ctx->family, - ip2str(&frontend_k.addr, (ctx->family == AF_INET)), + ip2str((__u32 *)&frontend_k.addr, (ctx->family == AF_INET)), bpf_ntohs(ctx->user_port)); frontend_v = map_lookup_frontend(&frontend_k); @@ -39,7 +40,7 @@ static inline int sock_traffic_control(struct kmesh_context *kmesh_ctx) KMESH, "bpf find frontend addr=[%u:%s:%u]\n", ctx->family, - ip2str(&kmesh_ctx->orig_dst_addr, (ctx->family == AF_INET)), + ip2str((__u32 *)&kmesh_ctx->orig_dst_addr, (ctx->family == AF_INET)), bpf_ntohs(ctx->user_port)); ret = frontend_manager(kmesh_ctx, frontend_v); if (ret != 0) { @@ -81,6 +82,8 @@ int cgroup_connect4_prog(struct bpf_sock_addr *ctx) // if tail call failed will run this code BPF_LOG(ERR, KMESH, "workload tail call failed, err is %d\n", ret); } + + observe_on_pre_connect(ctx->sk); return CGROUP_SOCK_OK; } @@ -124,6 +127,8 @@ int cgroup_connect6_prog(struct bpf_sock_addr *ctx) // if tail call failed will run this code BPF_LOG(ERR, KMESH, "workload tail call6 failed, err is %d\n", ret); } + + observe_on_pre_connect(ctx->sk); return CGROUP_SOCK_OK; } diff --git a/bpf/kmesh/workload/include/backend.h b/bpf/kmesh/workload/include/backend.h index 5f7aaa494..99037567c 100644 --- a/bpf/kmesh/workload/include/backend.h +++ b/bpf/kmesh/workload/include/backend.h @@ -61,7 +61,7 @@ backend_manager(struct kmesh_context *kmesh_ctx, backend_value *backend_v, __u32 DEBUG, BACKEND, "find waypoint addr=[%s:%u]\n", - ip2str(&backend_v->wp_addr, ctx->family == AF_INET), + ip2str((__u32 *)&backend_v->wp_addr, ctx->family == AF_INET), bpf_ntohs(backend_v->waypoint_port)); ret = waypoint_manager(kmesh_ctx, &backend_v->wp_addr, backend_v->waypoint_port); if (ret != 0) { @@ -71,10 +71,10 @@ backend_manager(struct kmesh_context *kmesh_ctx, backend_value *backend_v, __u32 } #pragma unroll - for (__u32 i = 0; i < backend_v->service_count; i++) { - if (i >= MAX_PORT_COUNT) { - BPF_LOG(WARN, BACKEND, "exceed the max port count:%d", MAX_PORT_COUNT); - return -EINVAL; + for (__u32 i = 0; i < MAX_SERVICE_COUNT; i++) { + if (i >= backend_v->service_count) { + BPF_LOG(WARN, BACKEND, "exceed the max backend service count:%d", backend_v->service_count); + break; } if (service_id == backend_v->service[i]) { BPF_LOG(DEBUG, BACKEND, "access the backend by service:%u\n", service_id); @@ -91,7 +91,7 @@ backend_manager(struct kmesh_context *kmesh_ctx, backend_value *backend_v, __u32 DEBUG, BACKEND, "get the backend addr=[%s:%u]\n", - ip2str(&kmesh_ctx->dnat_ip, ctx->family == AF_INET), + ip2str((__u32 *)&kmesh_ctx->dnat_ip, ctx->family == AF_INET), bpf_ntohs(service_v->target_port[j])); return 0; } diff --git a/bpf/kmesh/workload/include/config.h b/bpf/kmesh/workload/include/config.h index 9a2a386a4..a96a10faa 100644 --- a/bpf/kmesh/workload/include/config.h +++ b/bpf/kmesh/workload/include/config.h @@ -5,10 +5,10 @@ #define _KMESH_CONFIG_H_ // map size -#define MAP_SIZE_OF_FRONTEND 100 -#define MAP_SIZE_OF_SERVICE 100 -#define MAP_SIZE_OF_ENDPOINT 1000 -#define MAP_SIZE_OF_BACKEND 500 +#define MAP_SIZE_OF_FRONTEND 105000 +#define MAP_SIZE_OF_SERVICE 5000 +#define MAP_SIZE_OF_ENDPOINT 105000 +#define MAP_SIZE_OF_BACKEND 100000 #define MAP_SIZE_OF_AUTH 8192 #define MAP_SIZE_OF_DSTINFO 8192 diff --git a/bpf/kmesh/workload/include/encoder.h b/bpf/kmesh/workload/include/encoder.h index 879d64401..88c375aa2 100644 --- a/bpf/kmesh/workload/include/encoder.h +++ b/bpf/kmesh/workload/include/encoder.h @@ -12,7 +12,7 @@ struct { __type(key, __u64); __type(value, struct bpf_sock_tuple); __uint(max_entries, MAP_SIZE_OF_DSTINFO); - __uint(map_flags, 0); + __uint(map_flags, BPF_F_NO_PREALLOC); } map_of_dst_info SEC(".maps"); #endif /*__ENCODER_H__*/ \ No newline at end of file diff --git a/bpf/kmesh/workload/include/frontend.h b/bpf/kmesh/workload/include/frontend.h index fbfb4d1e8..4c1961340 100644 --- a/bpf/kmesh/workload/include/frontend.h +++ b/bpf/kmesh/workload/include/frontend.h @@ -41,7 +41,7 @@ static inline int frontend_manager(struct kmesh_context *kmesh_ctx, frontend_val DEBUG, FRONTEND, "find waypoint addr=[%s:%u]\n", - ip2str(&backend_v->wp_addr, kmesh_ctx->ctx->family == AF_INET), + ip2str((__u32 *)&backend_v->wp_addr, kmesh_ctx->ctx->family == AF_INET), bpf_ntohs(backend_v->waypoint_port)); ret = waypoint_manager(kmesh_ctx, &backend_v->wp_addr, backend_v->waypoint_port); if (ret != 0) { diff --git a/bpf/kmesh/workload/include/service.h b/bpf/kmesh/workload/include/service.h index 762aa3de3..e1500c3ae 100644 --- a/bpf/kmesh/workload/include/service.h +++ b/bpf/kmesh/workload/include/service.h @@ -46,7 +46,7 @@ static inline int service_manager(struct kmesh_context *kmesh_ctx, __u32 service DEBUG, SERVICE, "find waypoint addr=[%s:%u]\n", - ip2str(&service_v->wp_addr, kmesh_ctx->ctx->family == AF_INET), + ip2str((__u32 *)&service_v->wp_addr, kmesh_ctx->ctx->family == AF_INET), bpf_ntohs(service_v->waypoint_port)); ret = waypoint_manager(kmesh_ctx, &service_v->wp_addr, service_v->waypoint_port); if (ret != 0) { diff --git a/bpf/kmesh/workload/include/workload.h b/bpf/kmesh/workload/include/workload.h index 68c0e7617..333babd1e 100644 --- a/bpf/kmesh/workload/include/workload.h +++ b/bpf/kmesh/workload/include/workload.h @@ -10,19 +10,20 @@ #define MAX_SERVICE_COUNT 10 #define RINGBUF_SIZE (1 << 12) +#pragma pack(1) // frontend map typedef struct { struct ip_addr addr; // Service ip or Pod ip -} __attribute__((packed)) frontend_key; +} frontend_key; typedef struct { __u32 upstream_id; // service id for Service access or backend uid for Pod access -} __attribute__((packed)) frontend_value; +} frontend_value; // service map typedef struct { __u32 service_id; // service id -} __attribute__((packed)) service_key; +} service_key; typedef struct { __u32 endpoint_count; // endpoint count of current service @@ -32,30 +33,30 @@ typedef struct { __u32 target_port[MAX_PORT_COUNT]; struct ip_addr wp_addr; __u32 waypoint_port; -} __attribute__((packed)) service_value; +} service_value; // endpoint map typedef struct { __u32 service_id; // service id __u32 backend_index; // if endpoint_count = 3, then backend_index = 0/1/2 -} __attribute__((packed)) endpoint_key; +} endpoint_key; typedef struct { __u32 backend_uid; // workload_uid to uint32 -} __attribute__((packed)) endpoint_value; +} endpoint_value; // backend map typedef struct { __u32 backend_uid; // workload_uid to uint32 -} __attribute__((packed)) backend_key; - +} backend_key; typedef struct { struct ip_addr addr; __u32 service_count; __u32 service[MAX_SERVICE_COUNT]; struct ip_addr wp_addr; __u32 waypoint_port; -} __attribute__((packed)) backend_value; +} backend_value; +#pragma pack() struct { __uint(type, BPF_MAP_TYPE_HASH); @@ -94,6 +95,7 @@ struct { __type(key, struct bpf_sock_tuple); __type(value, __u32); __uint(max_entries, MAP_SIZE_OF_AUTH); + __uint(map_flags, BPF_F_NO_PREALLOC); } map_of_auth SEC(".maps"); struct { diff --git a/bpf/kmesh/workload/include/workload_common.h b/bpf/kmesh/workload/include/workload_common.h index e0ce6fee7..bcc92418f 100644 --- a/bpf/kmesh/workload/include/workload_common.h +++ b/bpf/kmesh/workload/include/workload_common.h @@ -23,10 +23,11 @@ typedef enum { LB_POLICY_RANDOM = 0, } lb_policy_t; +#pragma pack(1) typedef struct { __u32 protocol; __u32 ipv4; __u32 port; -} __attribute__((packed)) address_t; - +} address_t; +#pragma pack() #endif // _WORKLOAD_COMMON_H_ diff --git a/bpf/kmesh/workload/sockops.c b/bpf/kmesh/workload/sockops.c index 77482a100..0826624dc 100644 --- a/bpf/kmesh/workload/sockops.c +++ b/bpf/kmesh/workload/sockops.c @@ -11,6 +11,7 @@ #include "config.h" #include "encoder.h" #include "bpf_common.h" +#include "probe.h" #define FORMAT_IP_LENGTH (16) @@ -148,7 +149,7 @@ static inline void record_kmesh_managed_ip(__u32 family, __u32 ip4, __u32 *ip6) if (family == AF_INET6 && ip6) IP6_COPY(key.addr.ip6, ip6); - err = bpf_map_update_elem(&map_of_manager, &key, &value, BPF_NOEXIST); + err = bpf_map_update_elem(&map_of_manager, &key, &value, BPF_ANY); if (err) BPF_LOG(ERR, KMESH, "record ip failed!, err is %d\n", err); } @@ -177,7 +178,7 @@ static inline bool conn_from_sim(struct bpf_sock_ops *skops, __u32 ip, __u16 por return ( skops->remote_ip6[0] == 0 && skops->remote_ip6[1] == 0 && skops->remote_ip6[2] == 0 - && bpf_ntohl(skops->remote_ip6[3] == ip)); + && bpf_ntohl(skops->remote_ip6[3]) == ip); } static inline bool skops_conn_from_cni_sim_add(struct bpf_sock_ops *skops) @@ -256,6 +257,7 @@ int sockops_prog(struct bpf_sock_ops *skops) case BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB: if (!is_managed_by_kmesh(skops)) break; + observe_on_connect_established(skops->sk, OUTBOUND); if (bpf_sock_ops_cb_flags_set(skops, BPF_SOCK_OPS_STATE_CB_FLAG) != 0) BPF_LOG(ERR, SOCKOPS, "set sockops cb failed!\n"); __u64 *current_sk = (__u64 *)skops->sk; @@ -266,12 +268,14 @@ int sockops_prog(struct bpf_sock_ops *skops) case BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB: if (!is_managed_by_kmesh(skops)) break; + observe_on_connect_established(skops->sk, INBOUND); if (bpf_sock_ops_cb_flags_set(skops, BPF_SOCK_OPS_STATE_CB_FLAG) != 0) BPF_LOG(ERR, SOCKOPS, "set sockops cb failed!\n"); auth_ip_tuple(skops); break; case BPF_SOCK_OPS_STATE_CB: if (skops->args[1] == BPF_TCP_CLOSE) { + observe_on_close(skops->sk); clean_auth_map(skops); clean_dstinfo_map(skops); } diff --git a/bpf/kmesh/workload/xdp.c b/bpf/kmesh/workload/xdp.c index c9417eee7..dc9963d0d 100644 --- a/bpf/kmesh/workload/xdp.c +++ b/bpf/kmesh/workload/xdp.c @@ -68,7 +68,7 @@ static inline int should_shutdown(struct xdp_info *info, struct bpf_sock_tuple * INFO, XDP, "auth denied, src ip: %s, port: %u\n", - ip2str(&tuple_info->ipv6.saddr, false), + ip2str(&tuple_info->ipv6.saddr[0], false), bpf_ntohs(tuple_info->ipv6.sport)); bpf_map_delete_elem(&map_of_auth, tuple_info); return AUTH_FORBID; @@ -78,8 +78,8 @@ static inline int should_shutdown(struct xdp_info *info, struct bpf_sock_tuple * static inline int parser_xdp_info(struct xdp_md *ctx, struct xdp_info *info) { - void *begin = (void *)(ctx->data); - void *end = (void *)(ctx->data_end); + void *begin = (void *)(long)(ctx->data); + void *end = (void *)(long)(ctx->data_end); // eth header info->ethh = (struct ethhdr *)begin; diff --git a/build/docker/kmesh.dockerfile b/build/docker/kmesh.dockerfile index f5ba950a1..9cc007416 100644 --- a/build/docker/kmesh.dockerfile +++ b/build/docker/kmesh.dockerfile @@ -9,9 +9,13 @@ ARG arch RUN \ --mount=type=cache,target=/var/cache/dnf \ - yum install -y kmod util-linux iptables + yum install -y kmod util-linux iptables && \ + mkdir -p /usr/share/oncn-mda && \ + mkdir -p /etc/oncn-mda COPY out/$arch/*so* /usr/lib64/ +COPY out/$arch/*.o /usr/share/oncn-mda/ +COPY out/$arch/oncn-mda.conf /etc/oncn-mda/ COPY out/$arch/kmesh-daemon /usr/bin/ COPY out/$arch/kmesh-cni /usr/bin/ COPY out/$arch/mdacore /usr/bin/ diff --git a/daemon/manager/log/log.go b/daemon/manager/log/log.go index 0e28d8d43..5e359a916 100644 --- a/daemon/manager/log/log.go +++ b/daemon/manager/log/log.go @@ -27,7 +27,6 @@ import ( "github.com/spf13/cobra" - "kmesh.net/kmesh/pkg/logger" "kmesh.net/kmesh/pkg/status" ) @@ -49,32 +48,17 @@ func NewCmd() *cobra.Command { return cmd } -func GetLoggerLevel(args []string) { - if len(args) != 1 { - names := logger.GetLoggerNames() - if len(names) > 0 { - fmt.Println("Existing loggers:") - for _, name := range names { - fmt.Println(name) - } - } else { - fmt.Println("No existing loggers.") - } - return - } - loggerName := args[0] - url := status.GetLoggerURL() + "?name=" + loggerName - +func GetJson(url string, val any) { resp, err := http.Get(url) if err != nil { - fmt.Printf("Error making GET request: %v\n", err) + fmt.Printf("Error making GET request(%s): %v\n", url, err) return } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { - fmt.Printf("Error reading response body: %v\n", err) + fmt.Printf("Error reading response body(%s): %v\n", url, err) return } @@ -84,12 +68,32 @@ func GetLoggerLevel(args []string) { return } - var loggerInfo status.LoggerInfo - err = json.Unmarshal(body, &loggerInfo) + err = json.Unmarshal(body, val) if err != nil { fmt.Printf("Error unmarshaling response body: %v\n", err) return } +} + +func GetLoggerNames() { + url := status.GetLoggerURL() + var loggerNames []string + GetJson(url, &loggerNames) + fmt.Printf("Existing Loggers:\n") + for _, logger := range loggerNames { + fmt.Printf("\t%s\n", logger) + } +} + +func GetLoggerLevel(args []string) { + if len(args) == 0 { + GetLoggerNames() + return + } + loggerName := args[0] + url := status.GetLoggerURL() + "?name=" + loggerName + var loggerInfo status.LoggerInfo + GetJson(url, &loggerInfo) fmt.Printf("Logger Name: %s\n", loggerInfo.Name) fmt.Printf("Logger Level: %s\n", loggerInfo.Level) diff --git a/daemon/manager/manager.go b/daemon/manager/manager.go index 825e7affe..63edf4303 100644 --- a/daemon/manager/manager.go +++ b/daemon/manager/manager.go @@ -80,8 +80,11 @@ func Execute(configs *options.BootstrapConfigs) error { log.Info("bpf Start successful") defer bpfLoader.Stop() + stopCh := make(chan struct{}) + defer close(stopCh) + c := controller.NewController(configs, bpfLoader.GetBpfKmeshWorkload(), configs.BpfConfig.BpfFsPath, configs.BpfConfig.EnableBpfLog) - if err := c.Start(); err != nil { + if err := c.Start(stopCh); err != nil { return err } log.Info("controller Start successful") diff --git a/deploy/helm/templates/l7-envoyfilter.yaml b/deploy/helm/templates/l7-envoyfilter.yaml index 07879b4ac..f9580cd9c 100644 --- a/deploy/helm/templates/l7-envoyfilter.yaml +++ b/deploy/helm/templates/l7-envoyfilter.yaml @@ -18,6 +18,12 @@ spec: protocol: TCP address: 0.0.0.0 port_value: 15019 + additional_addresses: + - address: + socket_address: + protocol: TCP + address: "::" + port_value: 15019 filter_chains: - filters: - name: envoy.filters.network.http_connection_manager diff --git a/deploy/yaml/kmesh.yaml b/deploy/yaml/kmesh.yaml index 446051a4c..75cead7cf 100644 --- a/deploy/yaml/kmesh.yaml +++ b/deploy/yaml/kmesh.yaml @@ -68,7 +68,7 @@ spec: image: ghcr.io/kmesh-net/kmesh:latest imagePullPolicy: IfNotPresent command: ["/bin/sh", "-c"] - args: ["./start_kmesh.sh --mode=workload"] + args: ["./start_kmesh.sh --mode=workload --enable-bypass=false --enable-bpf-log=true"] securityContext: privileged: true capabilities: diff --git a/deploy/yaml/l7-envoyfilter.yaml b/deploy/yaml/l7-envoyfilter.yaml index 07879b4ac..f9580cd9c 100644 --- a/deploy/yaml/l7-envoyfilter.yaml +++ b/deploy/yaml/l7-envoyfilter.yaml @@ -18,6 +18,12 @@ spec: protocol: TCP address: 0.0.0.0 port_value: 15019 + additional_addresses: + - address: + socket_address: + protocol: TCP + address: "::" + port_value: 15019 filter_chains: - filters: - name: envoy.filters.network.http_connection_manager diff --git a/docs/kmesh_deploy_and_develop_in_kind.md b/docs/kmesh_deploy_and_develop_in_kind.md new file mode 100644 index 000000000..e0c65f353 --- /dev/null +++ b/docs/kmesh_deploy_and_develop_in_kind.md @@ -0,0 +1,152 @@ +# Kmesh Deploy and Develop in Kind + +[Kind](https://github.com/kubernetes-sigs/kind) is a convenient tool for quickly deploying kubernetes cluster locally. We can use `kind` to create an `istio` cluster and deploy `kmesh`. + +## Deploy Kmesh in Kind + +Let's start from setting up the required environment. You can follow the steps below: + ++ Install `kind`: + + Installing `kind` is very simple, because it's just a binary file. You can select the correct one according to the version and the architecture in the [github releases page](https://github.com/kubernetes-sigs/kind/releases). Take `linux` + `amd64` as example: + + ```shell + wget -O kind https://github.com/kubernetes-sigs/kind/releases/download/v0.23.0/kind-linux-amd64 + chmod +x kind + mv kind /usr/bin/ + ``` + ++ Create Kubernetes cluster using `kind`: + + You can take reference from the [istio official document](https://istio.io/latest/docs/setup/platform-setup/kind/). + + If you want to specified multiple workers or node image, you can: + + ```shell + kind create cluster --image=kindest/node:v1.23.17 --config=- </ string convert to uint32 variable -} __attribute__((packed)) service_key; +} service_key; typedef struct { @@ -98,25 +98,25 @@ typedef struct __u32 target_port[MAX_PORT_COUNT]; __u32 waypoint_addr; __u32 waypoint_port; -} __attribute__((packed)) service_value; +} service_value; // endpoint map typedef struct { __u32 service_id; // service id, through / string convert to uint32 variable __u32 backend_index; // backend index，The relationship of backend_index and endpoint_count：if endpoint_count is 3，then backend_index can be 1/2/3; -} __attribute__((packed)) endpoint_key; +} endpoint_key; typedef struct { __u32 backend_uid; // backend uid, through workload_uid string convert to uint32 variable -} __attribute__((packed)) endpoint_value; +} endpoint_value; // backend map typedef struct { __u32 backend_uid; // backend uid, through workload_uid string convert to uint32 variable -} __attribute__((packed)) backend_key; +} backend_key; typedef struct { @@ -124,7 +124,7 @@ typedef struct __u32 service[MAX_SERVICE_COUNT]; __u32 waypoint_addr; __u32 waypoint_port; -} __attribute__((packed)) backend_value; +} backend_value; ``` diff --git a/docs/proposal/pics/probe.svg b/docs/proposal/pics/probe.svg new file mode 100644 index 000000000..7ecd08e25 --- /dev/null +++ b/docs/proposal/pics/probe.svg @@ -0,0 +1,4 @@ + + + +
client
client
server
server
pre_connect
pre_connect
kmesh-daemon
kmesh-daemon
telemetry
telemetry
active_established
active_established
passive_established
passive_established
close
close
close
close
1
1
2
2
3
3
sock
sock
sk_storage
sk_storage
sock
sock
sk_storage
sk_storage
metrics
(BPF_MAP_TYPE_HASH)
metrics...
connect start time
connect start time
update direction
connect status
update direction...
statstic:
connect open
direction
statstic:...
4
4
5
5
statstic:
connect close
connect failed
sent_bytes
received_bytes
statstic:...
connect start time
connect start time
1
1
statstic:
connect open
direction
statstic:...
2
2
3
3
access_log
(BPF_MAP_TYPE_RINGBUF)
access_log...
statstic:
connect close
connect failed
sent_bytes
received_bytes
statstic:...
4
4Text is not SVG - cannot display \ No newline at end of file diff --git a/go.mod b/go.mod index 751d0bb65..9e9868477 100644 --- a/go.mod +++ b/go.mod @@ -6,14 +6,16 @@ go 1.22.0 replace github.com/imdario/mergo => github.com/imdario/mergo v0.3.5 require ( - github.com/agiledragon/gomonkey/v2 v2.11.0 + github.com/agiledragon/gomonkey/v2 v2.12.0 github.com/cespare/xxhash/v2 v2.3.0 github.com/cilium/ebpf v0.15.0 github.com/containernetworking/cni v1.2.2 github.com/containernetworking/plugins v1.5.1 github.com/envoyproxy/go-control-plane v0.12.1-0.20240628042141-fb8275a6aab4 github.com/golang/protobuf v1.5.4 - github.com/miekg/dns v1.1.59 + github.com/miekg/dns v1.1.61 + github.com/prometheus/client_golang v1.19.1 + github.com/prometheus/client_model v0.6.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 @@ -155,8 +157,6 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.19.1 // indirect - github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.54.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/prometheus/prometheus v0.52.1 // indirect diff --git a/go.sum b/go.sum index 237a1fbd3..f39365e5c 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= -github.com/agiledragon/gomonkey/v2 v2.11.0 h1:5oxSgA+tC1xuGsrIorR+sYiziYltmJyEZ9qA25b6l5U= -github.com/agiledragon/gomonkey/v2 v2.11.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= +github.com/agiledragon/gomonkey/v2 v2.12.0 h1:ek0dYu9K1rSV+TgkW5LvNNPRWyDZVIxGMCFI6Pz9o38= +github.com/agiledragon/gomonkey/v2 v2.12.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecholmes/xfccparser v0.3.0 h1:SI/zhgFw+CsoHnR2VXcbVg/9gij6T/ENT+1yqBOeLNA= github.com/alecholmes/xfccparser v0.3.0/go.mod h1:J9fzzUOtjw74IwNdGVbjnOVj1UDlwGQj1zZzgQRlRDY= @@ -478,8 +478,8 @@ github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU github.com/mdlayher/socket v0.1.1/go.mod h1:mYV5YIZAfHh4dzDVzI8x8tWLWCliuX8Mon5Awbj+qDs= github.com/mdlayher/socket v0.5.0 h1:ilICZmJcQz70vrWVes1MFera4jGiWNocSkykwwoy3XI= github.com/mdlayher/socket v0.5.0/go.mod h1:WkcBFfvyG8QENs5+hfQPl1X6Jpd2yeLIYgrGFmJiJxI= -github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs= -github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk= +github.com/miekg/dns v1.1.61 h1:nLxbwF3XxhwVSm8g9Dghm9MHPaUZuqhPiGL+675ZmEs= +github.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eykwQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= diff --git a/hack/copyright-check.sh b/hack/copyright-check.sh new file mode 100755 index 000000000..66dbdc957 --- /dev/null +++ b/hack/copyright-check.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +ROOT_DIR=$(git rev-parse --show-toplevel) + +go_copyright_path=$ROOT_DIR/hack/copyright/apache.txt + +c_copyright_path1=$ROOT_DIR/hack/copyright/BSDandGPL1.txt +c_copyright_path2=$ROOT_DIR/hack/copyright/BSDandGPL2.txt + +go_dirs="$ROOT_DIR/pkg" +c_dirs="$ROOT_DIR/bpf" + +function check_go_copyright() { + target_file=$1 + copyright_file=$go_copyright_path + + if [ ! -f "$target_file" ]; then + echo "Target file $target_file does not exist." + exit 1 + fi + + if [ ! -f "$copyright_file" ]; then + echo "Copyright file $copyright_file does not exist." + exit 1 + fi + + all_lines_present=true + while IFS= read -r line; do + if ! grep -qF -- "$line" "$target_file"; then + all_lines_present=false + break + fi + done < "$copyright_file" + + if [ "$all_lines_present" != true ]; then + echo "The target file does not contain all lines from the copyright file." + echo $target_file + fi +} + +function check_c_copyright() { + target_file=$1 + copyright_file1=$c_copyright_path1 + copyright_file2=$c_copyright_path2 + + if [ ! -f "$target_file" ]; then + echo "Target file $target_file does not exist." + exit 1 + fi + + if [ ! -f "$copyright_file1" ]; then + echo "Copyright file $copyright_file1 does not exist." + exit 1 + fi + + if [ ! -f "$copyright_file2" ]; then + echo "Copyright file $copyright_file2 does not exist." + exit 1 + fi + + all_lines_present1=true + while IFS= read -r line; do + if ! grep -qF -- "$line" "$target_file"; then + all_lines_present1=false + break + fi + done < "$copyright_file1" + + all_lines_present2=true + while IFS= read -r line; do + if ! grep -qF -- "$line" "$target_file"; then + all_lines_present2=false + break + fi + done < "$copyright_file2" + + if [ "$all_lines_present1" != true ] && [ "$all_lines_present2" != true ]; then + echo "The target file does not contain all lines from the copyright file." + echo $target_file + fi +} + +function go_check_dir() { + dir=$1 + find $dir -type f -name "*.go" | while read file; do + # echo $file + if ! echo $exclude_dirs | grep -q $(dirname $file); then + check_go_copyright $file + fi + done +} + +function c_check_dir() { + dir=$1 + find $dir -type f -name "*.c" -o -name "*.h" | while read file; do + # echo $file + if ! echo $exclude_dirs | grep -q $(dirname $file); then + check_c_copyright $file + fi + done +} + +function copyright_check() { + for dir in ${go_dirs}; do + go_check_dir $dir + done + + for dir in ${c_dirs}; do + c_check_dir $dir + done +} + +# Confirmation of whether there are files with wrong copyright +if [ -z "$1" -o "$1" == "-c" -o "$1" == "--check" ]; then + result=$(copyright_check | grep -c 'not contain') + if [ "$result" != 0 ]; then + echo "ERROR: Some files need to be update copyright, please run "./hack/copyright-check -g" and update and include any changed files in your PR" + exit 1 + else + echo "pass copyright check" + fi +fi + +# Obtaining the name of a file that with wrong copyright +if [ "$1" == "-g" -o "$1" == "--get" ]; then + copyright_check +fi diff --git a/hack/copyright/BSDandGPL1.txt b/hack/copyright/BSDandGPL1.txt new file mode 100644 index 000000000..ccaaa529b --- /dev/null +++ b/hack/copyright/BSDandGPL1.txt @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Copyright Authors of Kmesh */ \ No newline at end of file diff --git a/hack/copyright/BSDandGPL2.txt b/hack/copyright/BSDandGPL2.txt new file mode 100644 index 000000000..74d3b29eb --- /dev/null +++ b/hack/copyright/BSDandGPL2.txt @@ -0,0 +1,2 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* Copyright Authors of Kmesh */ \ No newline at end of file diff --git a/hack/copyright/apache.txt b/hack/copyright/apache.txt new file mode 100644 index 000000000..6e9499eb0 --- /dev/null +++ b/hack/copyright/apache.txt @@ -0,0 +1,15 @@ +/* + * Copyright The Kmesh Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ \ No newline at end of file diff --git a/hack/run-ut.sh b/hack/run-ut.sh index 940f31fb1..1602a4781 100755 --- a/hack/run-ut.sh +++ b/hack/run-ut.sh @@ -5,10 +5,14 @@ ROOT_DIR=$(git rev-parse --show-toplevel) . $ROOT_DIR/hack/utils.sh function get_go_test_command() { + if [ -z "$TEST_PKG" ]; then + TEST_PKG="./pkg/..." + fi + if [ -z "$TEST_TARGET" ]; then - echo "go test -v -race -vet=off ./pkg/..." + echo "go test -v -race -vet=off $TEST_PKG" else - echo "go test -v -race -vet=off -run $TEST_TARGET ./pkg/..." + echo "go test -v -race -vet=off -run $TEST_TARGET $TEST_PKG" fi } diff --git a/hack/utils.sh b/hack/utils.sh index cf83c7f11..5bb3b66f9 100644 --- a/hack/utils.sh +++ b/hack/utils.sh @@ -57,6 +57,9 @@ function copy_to_host() { cp /usr/lib64/libkmesh_api_v2_c.so out/$arch cp /usr/lib64/libkmesh_deserial.so out/$arch cp /usr/lib64/libboundscheck.so out/$arch + cp oncn-mda/build/ebpf_src/CMakeFiles/sock_redirect.dir/sock_redirect.c.o out/$arch + cp oncn-mda/etc/oncn-mda.conf out/$arch + cp oncn-mda/build/ebpf_src/CMakeFiles/sock_ops.dir/sock_ops.c.o out/$arch find /usr/lib64 -name 'libbpf.so*' -exec cp {} out/$arch \; find /usr/lib64 -name 'libprotobuf-c.so*' -exec cp {} out/$arch \; cp /usr/bin/kmesh-daemon out/$arch diff --git a/kmesh.spec b/kmesh.spec index caa9ec773..9da38ad2f 100644 --- a/kmesh.spec +++ b/kmesh.spec @@ -32,7 +32,6 @@ cd %{_builddir}/%{name}-%{version} %install mkdir -p %{buildroot}%{_bindir} install %{_builddir}/%{name}-%{version}/kmesh-daemon %{buildroot}%{_bindir} -install %{_builddir}/%{name}-%{version}/kmesh-cmd %{buildroot}%{_bindir} install %{_builddir}/%{name}-%{version}/kmesh-cni %{buildroot}%{_bindir} install %{_builddir}/%{name}-%{version}/build/kmesh-start-pre.sh %{buildroot}%{_bindir} install %{_builddir}/%{name}-%{version}/build/kmesh-stop-post.sh %{buildroot}%{_bindir} @@ -89,7 +88,6 @@ rm -rf %{buildroot} %files %defattr(-,root,root) %attr(0500,root,root) %{_bindir}/kmesh-daemon -%attr(0500,root,root) %{_bindir}/kmesh-cmd %attr(0500,root,root) %{_bindir}/kmesh-cni %attr(0500,root,root) %{_bindir}/mdacore @@ -105,7 +103,6 @@ rm -rf %{buildroot} %attr(0700,root,root) %dir %{_sysconfdir}/kmesh %attr(0700,root,root) %dir %{_sysconfdir}/oncn-mda -%config(noreplace) %attr(0600,root,root) %{_sysconfdir}/kmesh/kmesh.json %config(noreplace) %attr(0600,root,root) %{_sysconfdir}/oncn-mda/oncn-mda.conf %config(noreplace) %attr(0600,root,root) /usr/lib/systemd/system/kmesh.service %attr(0500,root,root) /usr/bin/kmesh-start-pre.sh @@ -113,4 +110,4 @@ rm -rf %{buildroot} %changelog * Wed Sep 27 2023 kwb0523 - 0.0.1-1 -- init package +- init package \ No newline at end of file diff --git a/oncn-mda/cli_src/func/global.c b/oncn-mda/cli_src/func/global.c index 54d421c6c..2579078e2 100644 --- a/oncn-mda/cli_src/func/global.c +++ b/oncn-mda/cli_src/func/global.c @@ -22,7 +22,11 @@ const char pinprog_file_path[][PATH_MAX] = { "/sys/fs/bpf/meshAccelerate/sock_redirect", }; +#if LIBBPF_HIGHER_0_6_0_VERSION +struct create_map_attr g_sock_param_map_xattr = { +#else struct bpf_create_map_attr g_sock_param_map_xattr = { +#endif .name = to_str(SOCK_PARAM_MAP_NAME), .map_type = BPF_MAP_TYPE_ARRAY, .key_size = sizeof(__u32), @@ -30,7 +34,11 @@ struct bpf_create_map_attr g_sock_param_map_xattr = { .max_entries = 1, }; +#if LIBBPF_HIGHER_0_6_0_VERSION +struct create_map_attr g_sock_ops_map_xattr = { +#else struct bpf_create_map_attr g_sock_ops_map_xattr = { +#endif .name = to_str(SOCK_OPS_MAP_NAME), .map_type = BPF_MAP_TYPE_SOCKHASH, .key_size = sizeof(struct sock_key), @@ -39,7 +47,11 @@ struct bpf_create_map_attr g_sock_ops_map_xattr = { }; #if MDA_GID_UID_FILTER +#if LIBBPF_HIGHER_0_6_0_VERSION +struct create_map_attr g_sock_ops_helper_map_xattr = { +#else struct bpf_create_map_attr g_sock_ops_helper_map_xattr = { +#endif .name = to_str(SOCK_OPS_HELPER_MAP_NAME), .map_type = BPF_MAP_TYPE_HASH, .key_size = sizeof(struct sock_key), @@ -48,7 +60,11 @@ struct bpf_create_map_attr g_sock_ops_helper_map_xattr = { }; #endif +#if LIBBPF_HIGHER_0_6_0_VERSION +struct create_map_attr g_sock_ops_proxy_map_xattr = { +#else struct bpf_create_map_attr g_sock_ops_proxy_map_xattr = { +#endif .name = to_str(SOCK_OPS_PROXY_MAP_NAME), .map_type = BPF_MAP_TYPE_HASH, .key_size = sizeof(struct sock_key), @@ -56,7 +72,11 @@ struct bpf_create_map_attr g_sock_ops_proxy_map_xattr = { .max_entries = SKOPS_MAP_SIZE, }; +#if LIBBPF_HIGHER_0_6_0_VERSION +struct create_map_attr g_sock_dump_map = { +#else struct bpf_create_map_attr g_sock_dump_map = { +#endif .name = to_str(SOCK_DUMP_MAP_I_NAME), .map_type = BPF_MAP_TYPE_QUEUE, .key_size = 0, @@ -64,7 +84,11 @@ struct bpf_create_map_attr g_sock_dump_map = { .max_entries = DUMP_QUEUE_LENGTH, }; +#if LIBBPF_HIGHER_0_6_0_VERSION +struct create_map_attr g_sock_dump_data_map = { +#else struct bpf_create_map_attr g_sock_dump_data_map = { +#endif .name = to_str(SOCK_DUMP_CPU_ARRAY_NAME), .map_type = BPF_MAP_TYPE_PERCPU_ARRAY, .key_size = sizeof(__u32), @@ -72,12 +96,12 @@ struct bpf_create_map_attr g_sock_dump_data_map = { .max_entries = 1, }; -struct bpf_object_open_attr g_sock_ops_xattr = { +struct object_open_attr g_sock_ops_xattr = { .prog_type = BPF_PROG_TYPE_SOCK_OPS, .file = SOCK_OPS_PATH_INIT, }; -struct bpf_object_open_attr g_sock_redirect_xattr = { +struct object_open_attr g_sock_redirect_xattr = { .prog_type = BPF_PROG_TYPE_SK_MSG, .file = SOCK_REDIRECT_PATH_INIT, }; @@ -162,7 +186,11 @@ static int init_mesh_map( struct mesh_map_info *const fds_map, const char *const pin_file_path, const char *const map_name, +#if LIBBPF_HIGHER_0_6_0_VERSION + struct create_map_attr *const map_attr) +#else struct bpf_create_map_attr *const map_attr) +#endif { int ret = EOK; ret += strcpy_s(fds_map->name, BPF_OBJ_NAME_LEN, map_name); @@ -180,7 +208,7 @@ static int init_mesh_map( static int init_mesh_prog( struct mesh_prog_info *const fds_prog, const char *const prog_name, - struct bpf_object_open_attr *const prog_attr, + struct object_open_attr *const prog_attr, enum bpf_attach_type attach_type, int attach_fd) { diff --git a/oncn-mda/cli_src/func/switch.c b/oncn-mda/cli_src/func/switch.c index e2dfe3f3c..416f00e90 100644 --- a/oncn-mda/cli_src/func/switch.c +++ b/oncn-mda/cli_src/func/switch.c @@ -67,7 +67,7 @@ static struct bpf_object *get_program_object(struct mesh_prog_info *const prog_i return NULL; } - struct bpf_object *obj = bpf_object__open_xattr(prog_info->xattr); + struct bpf_object *obj = bpf_object__open_file(prog_info->xattr->file, NULL); if (obj == NULL) { macli_log(ERR, "can not open bpf program, path:%s, errno:%d\n", prog_info->xattr->file, errno); return NULL; @@ -79,7 +79,11 @@ static int set_program_type(const struct mesh_prog_info *const prog_info, const { enum bpf_prog_type prog_type = prog_info->xattr->prog_type; enum bpf_attach_type expected_attach_type = prog_info->attach_type; +#if LIBBPF_HIGHER_0_6_0_VERSION + struct bpf_program *pos = bpf_object__next_program(obj, NULL); +#else struct bpf_program *pos = bpf_program__next(NULL, obj); +#endif if (pos == NULL) { macli_log(ERR, "obj:%s not contain a ebpf program!\n", prog_info->xattr->file); return FAILED; @@ -113,7 +117,11 @@ static int pinned_program_file(const struct mesh_prog_info *const prog_info, str macli_log(ERR, "open bpf obj:%s failed! errno:%d\n", prog_info->xattr->file, errno); return FAILED; } +#if LIBBPF_HIGHER_0_6_0_VERSION + prog = bpf_object__next_program(obj, NULL); +#else prog = bpf_program__next(NULL, obj); +#endif if (!prog) { macli_log(ERR, "object file:%s doesn't contain any bpf program\n", prog_info->xattr->file); return FAILED; @@ -156,7 +164,17 @@ static int create_map(struct mesh_service_info *const fds) { for (unsigned int i = 0; i < MESH_MAP_NUM; ++i) { if (fds->map_fds[i].fd == -1) { +#if LIBBPF_HIGHER_0_6_0_VERSION + fds->map_fds[i].fd = bpf_map_create( + fds->map_fds[i].xattr->map_type, + fds->map_fds[i].xattr->name, + fds->map_fds[i].xattr->key_size, + fds->map_fds[i].xattr->value_size, + fds->map_fds[i].xattr->max_entries, + NULL); +#else fds->map_fds[i].fd = bpf_create_map_xattr(fds->map_fds[i].xattr); +#endif if (fds->map_fds[i].fd < 0) { macli_log(ERR, "create %s failed! errno:%d\n", fds->map_fds[i].name, errno); return FAILED; diff --git a/oncn-mda/ebpf_src/sock_ops.c b/oncn-mda/ebpf_src/sock_ops.c index 7b9480810..f5ab001f8 100644 --- a/oncn-mda/ebpf_src/sock_ops.c +++ b/oncn-mda/ebpf_src/sock_ops.c @@ -141,13 +141,20 @@ filter_gid(const struct uid_gid_info *const current_uid_gid, const struct input_ return !filter_flag; } -static void get_current_uid_gid(struct uid_gid_info *const current_uid_gid, struct bpf_sock_ops *const skops) +static int get_current_uid_gid(struct uid_gid_info *const current_uid_gid, struct bpf_sock_ops *const skops) { - __u64 uid_gid = bpf_get_sockops_uid_gid(skops); + __u64 uid_gid; + int uid_gid_len = sizeof(uid_gid); + int ret = bpf_getsockopt(skops, SOL_IP, SK_BPF_GID_UID, &uid_gid, uid_gid_len); + if (ret < 0) { + bpf_log(ERROR, "bpf_getsockopt bpf_sock_ops_get_uid_gid failed, ret = %u\n", ret); + return ret; + } current_uid_gid->cuid = (uid_gid & 0xffffffff); current_uid_gid->cgid = (uid_gid >> UID_LENGTH); current_uid_gid->puid = 0; current_uid_gid->pgid = 0; + return 0; } static int get_peer_uid_gid(const struct sock_key *const peer_key, struct uid_gid_info *const current_uid_gid) @@ -190,7 +197,10 @@ filter(const struct sock_key *const key, const struct sock_key *const peer_key, return FILTER_PASS; struct uid_gid_info current_uid_gid = {0}; - get_current_uid_gid(¤t_uid_gid, skops); + if (get_current_uid_gid(¤t_uid_gid, skops) < 0) { + return FILTER_RETURN; + } + if (get_peer_uid_gid(peer_key, ¤t_uid_gid) != SUCCESS) { // The UID and GID of the peer are not found. mey be in other node bpf_log(INFO, "can not found the peer helper info! peer key:%u:%u\n", peer_key->sport, peer_key->sip4); @@ -262,7 +272,7 @@ get_peer_addr(struct bpf_sock_ops *const skops, const struct sock_key *const key #if MDA_NAT_ACCEL struct sockaddr_in target = {0}; int target_len = sizeof(target); - int ret = bpf_sk_original_addr(skops, SO_ORIGINAL_DST, (void *)&target, target_len); + int ret = bpf_getsockopt(skops, SOL_IP, BPF_SO_ORIGINAL_DST, &target, target_len); if (ret == 0) { peer_key->sip4 = key->dip4; peer_key->sport = key->dport; @@ -311,7 +321,9 @@ static void active_ops_ipv4(struct bpf_sock_ops *const skops) #if MDA_GID_UID_FILTER struct uid_gid_info current_uid_gid = {0}; - get_current_uid_gid(¤t_uid_gid, skops); + if (get_current_uid_gid(¤t_uid_gid, skops) < 0) { + return; + } if (add_helper_hash(&key, ¤t_uid_gid)) { bpf_log(ERROR, "active_ops_ipv4 failed!\n"); (void)clean_ops(&key); diff --git a/oncn-mda/include/data.h b/oncn-mda/include/data.h index 69fdbf4d7..951a3ab15 100644 --- a/oncn-mda/include/data.h +++ b/oncn-mda/include/data.h @@ -10,8 +10,10 @@ #define SKOPS_MAP_SIZE 196608 #define MAX_PARAM_LENGTH 10 -#define DUMP_QUEUE_LENGTH 4096 -#define MAX_DUMP_DATA_SIZE 4096 +#define DUMP_QUEUE_LENGTH 4096 +#define MAX_DUMP_DATA_SIZE 4096 +#define SK_BPF_GID_UID 18000 +#define BPF_SO_ORIGINAL_DST 800 #define SUCCESS 0 #define FAILED 1 @@ -40,6 +42,7 @@ #define to_str__(X) #X #define to_str(X) to_str__(X) +#pragma pack(1) struct sock_key { __u32 sip4; __u32 dip4; @@ -48,17 +51,17 @@ struct sock_key { #if MDA_LOOPBACK_ADDR __u64 netns_cookie; #endif -} __attribute__((packed)); +}; struct cidr { __u32 ip4; __u32 mask; -} __attribute__((packed)); +}; struct port_range { __u32 begin_port; __u32 end_port; -} __attribute__((packed)); +}; struct dump_prarm { __u32 switch_on; @@ -66,7 +69,8 @@ struct dump_prarm { __u8 current_port_num; // Indicates the number of ports to be filtered struct cidr dump_cidr[MAX_PARAM_LENGTH]; struct port_range dump_port[MAX_PARAM_LENGTH]; -} __attribute__((packed)); +}; + // When modifying the following structure, place PARAM_SIZE last enum ma_param_type { ACCEPT_IP = 0, @@ -88,12 +92,12 @@ enum ma_param_type { struct input_cidr { __u8 current_cidr_num; struct cidr cidrs[MAX_PARAM_LENGTH]; -} __attribute__((packed)); +}; struct input_port { __u8 current_port_num; struct port_range ports[MAX_PARAM_LENGTH]; -} __attribute__((packed)); +}; #if MDA_GID_UID_FILTER struct uid_gid_info { @@ -101,17 +105,17 @@ struct uid_gid_info { __u32 cgid; __u32 puid; __u32 pgid; -} __attribute__((packed)); +}; struct input_uid { __u8 current_uid_num; __u32 uids[MAX_PARAM_LENGTH]; -} __attribute__((packed)); +}; struct input_gid { __u8 current_gid_num; __u32 gids[MAX_PARAM_LENGTH]; -} __attribute__((packed)); +}; #endif struct sock_param { @@ -126,7 +130,7 @@ struct sock_param { struct input_gid return_gids; #endif struct dump_prarm dump_params; -} __attribute__((packed)); +}; struct dump_data { __u64 timestamp; @@ -137,4 +141,17 @@ struct dump_data { __u32 data_length; }; +#pragma pack() +struct create_map_attr { + const char *name; + enum bpf_map_type map_type; + __u32 key_size; + __u32 value_size; + __u32 max_entries; +}; + +struct object_open_attr { + const char *file; + enum bpf_prog_type prog_type; +}; #endif diff --git a/oncn-mda/include/macli.h b/oncn-mda/include/macli.h index f8b6edecf..ebe8f69e8 100644 --- a/oncn-mda/include/macli.h +++ b/oncn-mda/include/macli.h @@ -80,14 +80,18 @@ enum MESH_PROG { struct mesh_map_info { char name[BPF_OBJ_NAME_LEN]; char pin_file_path[PATH_MAX]; +#if LIBBPF_HIGHER_0_6_0_VERSION + struct create_map_attr *xattr; +#else struct bpf_create_map_attr *xattr; +#endif int fd; }; struct mesh_prog_info { char name[BPF_OBJ_NAME_LEN]; char pin_file_path[PATH_MAX]; - struct bpf_object_open_attr *xattr; + struct object_open_attr *xattr; enum bpf_attach_type attach_type; int attach_fd; int fd; diff --git a/oncn-mda/include/mesh_accelerate.h b/oncn-mda/include/mesh_accelerate.h index 5b81f02c0..39aeca01a 100644 --- a/oncn-mda/include/mesh_accelerate.h +++ b/oncn-mda/include/mesh_accelerate.h @@ -58,53 +58,53 @@ enum bpf_loglevel { #define LOOPBACK_ADDR 16777343 -struct bpf_map_def SEC("maps") SOCK_OPS_MAP_NAME = { - .type = BPF_MAP_TYPE_SOCKHASH, - .key_size = sizeof(struct sock_key), - .value_size = sizeof(int), - .max_entries = SKOPS_MAP_SIZE, - .map_flags = 0, -}; - -struct bpf_map_def SEC("maps") SOCK_PARAM_MAP_NAME = { - .type = BPF_MAP_TYPE_ARRAY, - .key_size = sizeof(__u32), - .value_size = sizeof(struct sock_param), - .max_entries = 1, -}; - -struct bpf_map_def SEC("maps") SOCK_OPS_PROXY_MAP_NAME = { - .type = BPF_MAP_TYPE_HASH, - .key_size = sizeof(struct sock_key), - .value_size = sizeof(struct sock_key), - .max_entries = SKOPS_MAP_SIZE, - .map_flags = 0, -}; +struct { + __uint(type, BPF_MAP_TYPE_SOCKHASH); + __type(key, struct sock_key); + __type(value, int); + __uint(max_entries, SKOPS_MAP_SIZE); + __uint(map_flags, 0); +} SOCK_OPS_MAP_NAME SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, __u32); + __type(value, struct sock_param); + __uint(max_entries, 1); +} SOCK_PARAM_MAP_NAME SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, struct sock_key); + __type(value, struct sock_key); + __uint(max_entries, SKOPS_MAP_SIZE); + __uint(map_flags, 0); +} SOCK_OPS_PROXY_MAP_NAME SEC(".maps"); #if MDA_GID_UID_FILTER -struct bpf_map_def SEC("maps") SOCK_OPS_HELPER_MAP_NAME = { - .type = BPF_MAP_TYPE_HASH, - .key_size = sizeof(struct sock_key), - .value_size = sizeof(struct uid_gid_info), - .max_entries = SKOPS_MAP_SIZE, - .map_flags = 0, -}; +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, struct sock_key); + __type(value, struct uid_gid_info); + __uint(max_entries, SKOPS_MAP_SIZE); + __uint(map_flags, 0); +} SOCK_OPS_HELPER_MAP_NAME SEC(".maps"); #endif -struct bpf_map_def SEC("maps") SOCK_DUMP_MAP_I_NAME = { - .type = BPF_MAP_TYPE_QUEUE, - .key_size = 0, - .value_size = sizeof(struct dump_data), - .max_entries = DUMP_QUEUE_LENGTH, - .map_flags = 0, -}; - -struct bpf_map_def SEC("maps") SOCK_DUMP_CPU_ARRAY_NAME = { - .type = BPF_MAP_TYPE_PERCPU_ARRAY, - .key_size = sizeof(__u32), - .value_size = sizeof(struct dump_data), - .max_entries = 1, -}; +struct { + __uint(type, BPF_MAP_TYPE_QUEUE); + __uint(key_size, 0); + __type(value, struct dump_data); + __uint(max_entries, DUMP_QUEUE_LENGTH); + __uint(map_flags, 0); +} SOCK_DUMP_MAP_I_NAME SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, __u32); + __type(value, struct dump_data); + __uint(max_entries, 1); +} SOCK_DUMP_CPU_ARRAY_NAME SEC(".maps"); #if MDA_LOOPBACK_ADDR static inline void set_netns_cookie(void *const ctx, struct sock_key *const key) diff --git a/pkg/auth/policy_store.go b/pkg/auth/policy_store.go index 1a8d77a5a..c5818cce9 100644 --- a/pkg/auth/policy_store.go +++ b/pkg/auth/policy_store.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/auth/policy_store_test.go b/pkg/auth/policy_store_test.go index a52df02d9..fcf5dfa2e 100644 --- a/pkg/auth/policy_store_test.go +++ b/pkg/auth/policy_store_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/auth/rbac.go b/pkg/auth/rbac.go index f76dece8f..35f7f3bd0 100644 --- a/pkg/auth/rbac.go +++ b/pkg/auth/rbac.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/auth/rbac_test.go b/pkg/auth/rbac_test.go index be02b62b0..710903fc8 100644 --- a/pkg/auth/rbac_test.go +++ b/pkg/auth/rbac_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/auth/xdp_auth_handler.go b/pkg/auth/xdp_auth_handler.go index 5d515f672..5fb0a40d7 100644 --- a/pkg/auth/xdp_auth_handler.go +++ b/pkg/auth/xdp_auth_handler.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/bpf/bpf.go b/pkg/bpf/bpf.go index 855e1fc67..764637291 100644 --- a/pkg/bpf/bpf.go +++ b/pkg/bpf/bpf.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/bpf/bpf_kmesh.go b/pkg/bpf/bpf_kmesh.go index e1a9e109e..390276f2c 100644 --- a/pkg/bpf/bpf_kmesh.go +++ b/pkg/bpf/bpf_kmesh.go @@ -2,7 +2,7 @@ // +build enhanced /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/bpf/bpf_kmesh_common.go b/pkg/bpf/bpf_kmesh_common.go index be28fa902..7c9f3bf28 100644 --- a/pkg/bpf/bpf_kmesh_common.go +++ b/pkg/bpf/bpf_kmesh_common.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/bpf/bpf_kmesh_l4.go b/pkg/bpf/bpf_kmesh_l4.go index b6ff9bc66..70fd6fa94 100644 --- a/pkg/bpf/bpf_kmesh_l4.go +++ b/pkg/bpf/bpf_kmesh_l4.go @@ -2,7 +2,7 @@ // +build !enhanced /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/bpf/bpf_kmesh_l4_workload.go b/pkg/bpf/bpf_kmesh_l4_workload.go index 826f49ea6..fb6fcd437 100644 --- a/pkg/bpf/bpf_kmesh_l4_workload.go +++ b/pkg/bpf/bpf_kmesh_l4_workload.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/bpf/bpf_kmesh_workload.go b/pkg/bpf/bpf_kmesh_workload.go index 5c508408e..e1e9ac7c1 100644 --- a/pkg/bpf/bpf_kmesh_workload.go +++ b/pkg/bpf/bpf_kmesh_workload.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/bpf/pin.go b/pkg/bpf/pin.go index e6ba762d0..24691da23 100644 --- a/pkg/bpf/pin.go +++ b/pkg/bpf/pin.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/cache/v2/cluster.go b/pkg/cache/v2/cluster.go index ddd95934e..357d5243e 100644 --- a/pkg/cache/v2/cluster.go +++ b/pkg/cache/v2/cluster.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/cache/v2/cluster_test.go b/pkg/cache/v2/cluster_test.go index 523941331..851c7d38b 100644 --- a/pkg/cache/v2/cluster_test.go +++ b/pkg/cache/v2/cluster_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/cache/v2/listener.go b/pkg/cache/v2/listener.go index cd58456a3..33fa9a85a 100644 --- a/pkg/cache/v2/listener.go +++ b/pkg/cache/v2/listener.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/cache/v2/listener_test.go b/pkg/cache/v2/listener_test.go index 0d052d03c..f4f26bda7 100644 --- a/pkg/cache/v2/listener_test.go +++ b/pkg/cache/v2/listener_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/cache/v2/maps/cluster.go b/pkg/cache/v2/maps/cluster.go index e18024c91..cd29529e5 100644 --- a/pkg/cache/v2/maps/cluster.go +++ b/pkg/cache/v2/maps/cluster.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/cache/v2/maps/common.go b/pkg/cache/v2/maps/common.go index ee193b304..f8f0fc1db 100644 --- a/pkg/cache/v2/maps/common.go +++ b/pkg/cache/v2/maps/common.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/cache/v2/maps/listener.go b/pkg/cache/v2/maps/listener.go index 2390c14d2..c1735414f 100644 --- a/pkg/cache/v2/maps/listener.go +++ b/pkg/cache/v2/maps/listener.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/cache/v2/maps/route.go b/pkg/cache/v2/maps/route.go index 5addf1f7e..60f3c6ef0 100644 --- a/pkg/cache/v2/maps/route.go +++ b/pkg/cache/v2/maps/route.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/cache/v2/route.go b/pkg/cache/v2/route.go index d5f35c588..929a12ec2 100644 --- a/pkg/cache/v2/route.go +++ b/pkg/cache/v2/route.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/cache/v2/route_test.go b/pkg/cache/v2/route_test.go index 4b1b79740..1dba73206 100644 --- a/pkg/cache/v2/route_test.go +++ b/pkg/cache/v2/route_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/cni/chained.go b/pkg/cni/chained.go index 252ea6d8e..5308674d8 100644 --- a/pkg/cni/chained.go +++ b/pkg/cni/chained.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/cni/chained_test.go b/pkg/cni/chained_test.go index c4e0f13c2..9c8efeb45 100644 --- a/pkg/cni/chained_test.go +++ b/pkg/cni/chained_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/cni/install.go b/pkg/cni/install.go index cca6d4ee4..bacefc710 100644 --- a/pkg/cni/install.go +++ b/pkg/cni/install.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/cni/kubeconfig.go b/pkg/cni/kubeconfig.go index e5863bbf2..531981495 100644 --- a/pkg/cni/kubeconfig.go +++ b/pkg/cni/kubeconfig.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/cni/plugin/plugin.go b/pkg/cni/plugin/plugin.go index 884e04f77..c3115ae36 100644 --- a/pkg/cni/plugin/plugin.go +++ b/pkg/cni/plugin/plugin.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/cni/plugin/plugin_test.go b/pkg/cni/plugin/plugin_test.go index a47eb859b..c751c4bc6 100644 --- a/pkg/cni/plugin/plugin_test.go +++ b/pkg/cni/plugin/plugin_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index b14d5b9e5..8bf24b05c 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,8 @@ const ( DataPlaneModeLabel = "istio.io/dataplane-mode" // DataPlaneModeKmesh is the value of the label to indicate the data plane mode is kmesh DataPlaneModeKmesh = "kmesh" + // This annotation is used to indicate traffic redirection settings specific to Kmesh + KmeshRedirectionAnnotation = "kmesh.net/redirection" XDP_PROG_NAME = "xdp_shutdown" @@ -46,4 +48,7 @@ const ( // tail call index in tail call prog map TailCallConnect4Index = 0 TailCallConnect6Index = 1 + + INBOUND = uint32(1) + OUTBOUND = uint32(2) ) diff --git a/pkg/controller/ads/ads_controller.go b/pkg/controller/ads/ads_controller.go index 4a59adcd7..c10e18364 100644 --- a/pkg/controller/ads/ads_controller.go +++ b/pkg/controller/ads/ads_controller.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/ads/ads_controller_test.go b/pkg/controller/ads/ads_controller_test.go index 1badf456c..a3e36425a 100644 --- a/pkg/controller/ads/ads_controller_test.go +++ b/pkg/controller/ads/ads_controller_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/ads/ads_processor.go b/pkg/controller/ads/ads_processor.go index 0a48d1e79..221516a72 100644 --- a/pkg/controller/ads/ads_processor.go +++ b/pkg/controller/ads/ads_processor.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/ads/ads_processor_test.go b/pkg/controller/ads/ads_processor_test.go index c1e4f22d3..4ba9c379e 100644 --- a/pkg/controller/ads/ads_processor_test.go +++ b/pkg/controller/ads/ads_processor_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/ads/cache.go b/pkg/controller/ads/cache.go index a945a5b3a..6e6f75229 100644 --- a/pkg/controller/ads/cache.go +++ b/pkg/controller/ads/cache.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/ads/cache_test.go b/pkg/controller/ads/cache_test.go index 0b3c5bf83..831016741 100644 --- a/pkg/controller/ads/cache_test.go +++ b/pkg/controller/ads/cache_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/ads/convert_filter.go b/pkg/controller/ads/convert_filter.go index f0749cfad..21cf0b80a 100644 --- a/pkg/controller/ads/convert_filter.go +++ b/pkg/controller/ads/convert_filter.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/ads/convert_filter_test.go b/pkg/controller/ads/convert_filter_test.go index 046d5fbe5..de6140e5d 100644 --- a/pkg/controller/ads/convert_filter_test.go +++ b/pkg/controller/ads/convert_filter_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/bypass/bypass_controller.go b/pkg/controller/bypass/bypass_controller.go index 7153d5a65..ee0a33b82 100644 --- a/pkg/controller/bypass/bypass_controller.go +++ b/pkg/controller/bypass/bypass_controller.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,14 +48,12 @@ const ( SidecarAnnotation = "sidecar.istio.io/inject" ) -func StartByPassController(client kubernetes.Interface) error { - stopChan := make(chan struct{}) +func StartByPassController(client kubernetes.Interface, stopChan <-chan struct{}) error { nodeName := os.Getenv("NODE_NAME") informerFactory := informers.NewSharedInformerFactoryWithOptions(client, DefaultInformerSyncPeriod, informers.WithTweakListOptions(func(options *metav1.ListOptions) { options.FieldSelector = fmt.Sprintf("spec.nodeName=%s", nodeName) - options.LabelSelector = LabelSelectorBypass })) informerFactory.Start(wait.NeverStop) @@ -70,6 +68,9 @@ func StartByPassController(client kubernetes.Interface) error { log.Errorf("expected *corev1.Pod but got %T", obj) return } + if !shouldEnroll(pod) { + return + } log.Infof("%s/%s: enable bypass control", pod.GetNamespace(), pod.GetName()) enableSidecar, _ := checkSidecar(client, pod) @@ -128,6 +129,8 @@ func StartByPassController(client kubernetes.Interface) error { } } }, + // We donot need to process delete here, because in bpf mode, it will be handled by kmesh-cni. + // In istio sidecar mode, we donot need to delete the iptables. }); err != nil { return fmt.Errorf("error adding event handler to podInformer: %v", err) } diff --git a/pkg/controller/bypass/bypass_test.go b/pkg/controller/bypass/bypass_test.go index f32a84382..cfd2eddcf 100644 --- a/pkg/controller/bypass/bypass_test.go +++ b/pkg/controller/bypass/bypass_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,7 +38,8 @@ func TestPodKmeshLabelChangeTriggersByPassKmeshAction(t *testing.T) { t.Cleanup(func() { os.Unsetenv("NODE_NAME") }) - err = StartByPassController(client) + stopCh := make(chan struct{}) + err = StartByPassController(client, stopCh) if err != nil { t.Fatalf("error creating ByPassController: %v", err) } @@ -109,7 +110,8 @@ func TestPodSidecarLabelChangeTriggersAddIptablesAction(t *testing.T) { t.Cleanup(func() { os.Unsetenv("NODE_NAME") }) - err = StartByPassController(client) + stopCh := make(<-chan struct{}) + err = StartByPassController(client, stopCh) if err != nil { t.Fatalf("error creating ByPassController: %v", err) } @@ -170,7 +172,7 @@ func TestPodSidecarLabelChangeTriggersAddIptablesAction(t *testing.T) { } wg.Add(1) - _, err = client.CoreV1().Pods("default").Create(context.TODO(), pod, metav1.CreateOptions{}) + _, err = client.CoreV1().Pods(namespaceName).Create(context.TODO(), pod, metav1.CreateOptions{}) assert.NoError(t, err) wg.Wait() @@ -179,9 +181,10 @@ func TestPodSidecarLabelChangeTriggersAddIptablesAction(t *testing.T) { enabled = false disabled = false - delete(pod.Labels, "kmesh.net/bypass") + newPod := pod.DeepCopy() + delete(newPod.Labels, "kmesh.net/bypass") wg.Add(1) - _, err = client.CoreV1().Pods("default").Update(context.TODO(), pod, metav1.UpdateOptions{}) + _, err = client.CoreV1().Pods(namespaceName).Update(context.TODO(), newPod, metav1.UpdateOptions{}) assert.NoError(t, err) wg.Wait() diff --git a/pkg/controller/client.go b/pkg/controller/client.go index c7e042c68..ae2d99e0e 100644 --- a/pkg/controller/client.go +++ b/pkg/controller/client.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/client_test.go b/pkg/controller/client_test.go index dd04fd30d..017e44200 100644 --- a/pkg/controller/client_test.go +++ b/pkg/controller/client_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/config/config.go b/pkg/controller/config/config.go index bd34d37db..adc2000fc 100644 --- a/pkg/controller/config/config.go +++ b/pkg/controller/config/config.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,7 +30,7 @@ import ( // in order to fix: could not resolve Any message type _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" _ "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" - _ "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" // nolint + _ "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" "kmesh.net/kmesh/pkg/constants" "kmesh.net/kmesh/pkg/logger" diff --git a/pkg/controller/config/config_test.go b/pkg/controller/config/config_test.go index 55c2a7f2c..6bb548adf 100644 --- a/pkg/controller/config/config_test.go +++ b/pkg/controller/config/config_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index e5b6825c5..2563815c2 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,8 +32,7 @@ import ( ) var ( - stopCh = make(chan struct{}) - ctx, cancle = context.WithCancel(context.Background()) + ctx, cancel = context.WithCancel(context.Background()) log = logger.NewLoggerField("controller") ) @@ -58,7 +57,7 @@ func NewController(opts *options.BootstrapConfigs, bpfWorkloadObj *bpf.BpfKmeshW } } -func (c *Controller) Start() error { +func (c *Controller) Start(stopCh <-chan struct{}) error { clientset, err := utils.GetK8sclient() if err != nil { return err @@ -68,12 +67,12 @@ func (c *Controller) Start() error { if err != nil { return fmt.Errorf("failed to start kmesh manage controller: %v", err) } - kmeshManageController.Run() + kmeshManageController.Run(stopCh) log.Info("start kmesh manage controller successfully") if c.enableByPass { - err = bypass.StartByPassController(clientset) + err = bypass.StartByPassController(clientset, stopCh) if err != nil { return fmt.Errorf("failed to start bypass controller: %v", err) } @@ -101,9 +100,7 @@ func (c *Controller) Start() error { go secertManager.Run(stopCh) c.client.WorkloadController.Processor.SecretManager = secertManager } - if c.client.WorkloadController.Rbac != nil { - go c.client.WorkloadController.Rbac.Run(c.client.ctx, c.bpfWorkloadObj.SockOps.MapOfTuple, c.bpfWorkloadObj.XdpAuth.MapOfAuth) - } + c.client.WorkloadController.Run(ctx) } if c.client.AdsController != nil { @@ -122,8 +119,7 @@ func (c *Controller) Stop() { if c == nil { return } - close(stopCh) - cancle() + cancel() if c.client != nil { c.client.Close() } diff --git a/pkg/controller/interfaces/client.go b/pkg/controller/interfaces/client.go index 11c1e7ffe..dcf3f1398 100644 --- a/pkg/controller/interfaces/client.go +++ b/pkg/controller/interfaces/client.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/manage/kmesh_manage.go b/pkg/controller/manage/kmesh_manage.go index ecc6c9ba2..26d2275f2 100644 --- a/pkg/controller/manage/kmesh_manage.go +++ b/pkg/controller/manage/kmesh_manage.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,21 +42,20 @@ var ( log = logger.NewLoggerField("manage_controller") annotationDelPatch = []byte(fmt.Sprintf( `{"metadata":{"annotations":{"%s":null}}}`, - KmeshRedirectionAnnotation, + constants.KmeshRedirectionAnnotation, )) annotationAddPatch = []byte(fmt.Sprintf( `{"metadata":{"annotations":{"%s":"%s"}}}`, - KmeshRedirectionAnnotation, + constants.KmeshRedirectionAnnotation, "enabled", )) ) const ( - DefaultInformerSyncPeriod = 30 * time.Second - MaxRetries = 5 - KmeshRedirectionAnnotation = "kmesh.net/redirection" - ActionAddAnnotation = "add" - ActionDeleteAnnotation = "delete" + DefaultInformerSyncPeriod = 30 * time.Second + MaxRetries = 5 + ActionAddAnnotation = "add" + ActionDeleteAnnotation = "delete" ) type QueueItem struct { @@ -65,7 +64,6 @@ type QueueItem struct { } func NewKmeshManageController(client kubernetes.Interface) (*KmeshManageController, error) { - stopChan := make(chan struct{}) nodeName := os.Getenv("NODE_NAME") informerFactory := informers.NewSharedInformerFactoryWithOptions(client, DefaultInformerSyncPeriod, @@ -84,7 +82,7 @@ func NewKmeshManageController(client kubernetes.Interface) (*KmeshManageControll return } - if !shouldEnroll(pod) { + if !shouldEnroll(client, pod) { return } @@ -112,7 +110,7 @@ func NewKmeshManageController(client kubernetes.Interface) (*KmeshManageControll } //add Kmesh manage label for enable Kmesh control - if !shouldEnroll(oldPod) && shouldEnroll(newPod) { + if !shouldEnroll(client, oldPod) && shouldEnroll(client, newPod) { log.Infof("%s/%s: enable Kmesh manage", newPod.GetNamespace(), newPod.GetName()) nspath, _ := ns.GetPodNSpath(newPod) @@ -125,7 +123,7 @@ func NewKmeshManageController(client kubernetes.Interface) (*KmeshManageControll } //delete Kmesh manage label for disable Kmesh control - if shouldEnroll(oldPod) && !shouldEnroll(newPod) { + if shouldEnroll(client, oldPod) && !shouldEnroll(client, newPod) { log.Infof("%s/%s: disable Kmesh manage", newPod.GetNamespace(), newPod.GetName()) nspath, _ := ns.GetPodNSpath(newPod) @@ -142,7 +140,6 @@ func NewKmeshManageController(client kubernetes.Interface) (*KmeshManageControll } return &KmeshManageController{ - stopChan: stopChan, informerFactory: informerFactory, podInformer: podInformer, queue: queue, @@ -151,17 +148,16 @@ func NewKmeshManageController(client kubernetes.Interface) (*KmeshManageControll } type KmeshManageController struct { - stopChan chan struct{} informerFactory informers.SharedInformerFactory podInformer cache.SharedIndexInformer queue workqueue.RateLimitingInterface client kubernetes.Interface } -func (c *KmeshManageController) Run() { +func (c *KmeshManageController) Run(stopChan <-chan struct{}) { go func() { - c.informerFactory.Start(c.stopChan) - if !cache.WaitForCacheSync(c.stopChan, c.podInformer.HasSynced) { + c.informerFactory.Start(stopChan) + if !cache.WaitForCacheSync(stopChan, c.podInformer.HasSynced) { log.Error("Timed out waiting for caches to sync") return } @@ -209,14 +205,30 @@ func isPodBeingDeleted(pod *corev1.Pod) bool { return pod.ObjectMeta.DeletionTimestamp != nil } -func shouldEnroll(pod *corev1.Pod) bool { - mode := pod.Labels[constants.DataPlaneModeLabel] - return strings.EqualFold(mode, constants.DataPlaneModeKmesh) +func shouldEnroll(client kubernetes.Interface, pod *corev1.Pod) bool { + // Check if the Pod's label indicates it should be managed by Kmesh + if strings.EqualFold(pod.Labels[constants.DataPlaneModeLabel], constants.DataPlaneModeKmesh) { + return true + } + if pod.Annotations[constants.KmeshRedirectionAnnotation] == "enabled" { + return true + } + + ns, err := client.CoreV1().Namespaces().Get(context.TODO(), pod.Namespace, metav1.GetOptions{}) + if err != nil { + log.Errorf("failed to get namespace %s: %v", pod.Namespace, err) + return false + } + // Check if the namespace's label indicates it should be managed by Kmesh + if strings.EqualFold(ns.Labels[constants.DataPlaneModeLabel], constants.DataPlaneModeKmesh) { + return true + } + return false } func addKmeshAnnotation(client kubernetes.Interface, pod *corev1.Pod) error { - if value, exists := pod.Annotations[KmeshRedirectionAnnotation]; exists && value == "enabled" { - log.Debugf("Pod %s in namespace %s already has annotation %s with value %s", pod.Name, pod.Namespace, KmeshRedirectionAnnotation, value) + if value, exists := pod.Annotations[constants.KmeshRedirectionAnnotation]; exists && value == "enabled" { + log.Debugf("Pod %s in namespace %s already has annotation %s with value %s", pod.Name, pod.Namespace, constants.KmeshRedirectionAnnotation, value) return nil } _, err := client.CoreV1().Pods(pod.Namespace).Patch( @@ -230,8 +242,8 @@ func addKmeshAnnotation(client kubernetes.Interface, pod *corev1.Pod) error { } func delKmeshAnnotation(client kubernetes.Interface, pod *corev1.Pod) error { - if _, exists := pod.Annotations[KmeshRedirectionAnnotation]; !exists { - log.Debugf("Pod %s in namespace %s does not have annotation %s", pod.Name, pod.Namespace, KmeshRedirectionAnnotation) + if _, exists := pod.Annotations[constants.KmeshRedirectionAnnotation]; !exists { + log.Debugf("Pod %s in namespace %s does not have annotation %s", pod.Name, pod.Namespace, constants.KmeshRedirectionAnnotation) return nil } _, err := client.CoreV1().Pods(pod.Namespace).Patch( diff --git a/pkg/controller/manage/kmesh_manage_test.go b/pkg/controller/manage/kmesh_manage_test.go index 03e546ea8..c2be13c56 100644 --- a/pkg/controller/manage/kmesh_manage_test.go +++ b/pkg/controller/manage/kmesh_manage_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -57,7 +57,7 @@ func TestPodWithLabelChangeTriggersManageAction(t *testing.T) { stopChan := make(chan struct{}) defer close(stopChan) - controller.Run() + controller.Run(stopChan) cache.WaitForCacheSync(stopChan, controller.podInformer.HasSynced) var mu sync.Mutex @@ -127,7 +127,7 @@ func TestPodWithoutLabelTriggersManageAction(t *testing.T) { stopChan := make(chan struct{}) defer close(stopChan) - controller.Run() + controller.Run(stopChan) cache.WaitForCacheSync(stopChan, controller.podInformer.HasSynced) var mu sync.Mutex diff --git a/pkg/controller/netns/netns.go b/pkg/controller/netns/netns.go index cf6a209c3..6cdcee62e 100644 --- a/pkg/controller/netns/netns.go +++ b/pkg/controller/netns/netns.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/security/caclient.go b/pkg/controller/security/caclient.go index 284c9f8bc..70f36e1b2 100644 --- a/pkg/controller/security/caclient.go +++ b/pkg/controller/security/caclient.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/security/caclient_test.go b/pkg/controller/security/caclient_test.go index 5c6b5559a..4a0e5bcfc 100644 --- a/pkg/controller/security/caclient_test.go +++ b/pkg/controller/security/caclient_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/security/manager.go b/pkg/controller/security/manager.go index 5e34d8c7e..6521664fb 100644 --- a/pkg/controller/security/manager.go +++ b/pkg/controller/security/manager.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/security/manager_test.go b/pkg/controller/security/manager_test.go index 3bd29bd6e..328740da4 100644 --- a/pkg/controller/security/manager_test.go +++ b/pkg/controller/security/manager_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/security/mock/mockcaclient.go b/pkg/controller/security/mock/mockcaclient.go index 9ac7b5ebf..f23aac785 100644 --- a/pkg/controller/security/mock/mockcaclient.go +++ b/pkg/controller/security/mock/mockcaclient.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/security/option.go b/pkg/controller/security/option.go index 8cf9b2ff3..42988f501 100644 --- a/pkg/controller/security/option.go +++ b/pkg/controller/security/option.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/telemetry/metric.go b/pkg/controller/telemetry/metric.go new file mode 100644 index 000000000..80bf00d15 --- /dev/null +++ b/pkg/controller/telemetry/metric.go @@ -0,0 +1,272 @@ +/* + * Copyright The Kmesh Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package telemetry + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "net/netip" + "reflect" + + "github.com/cilium/ebpf" + "github.com/cilium/ebpf/ringbuf" + + "kmesh.net/kmesh/api/v2/workloadapi" + "kmesh.net/kmesh/pkg/constants" + "kmesh.net/kmesh/pkg/controller/workload/cache" +) + +type MetricController struct { + workloadCache cache.WorkloadCache +} + +type metricKey struct { + SrcIp [4]uint32 + DstIp [4]uint32 +} + +type metricValue struct { + Direction uint32 + ConnectionOpen uint32 + ConnectionClose uint32 + ConnectionFailed uint32 + SentBytes uint32 + ReceivedBytes uint32 +} + +type requestMetric struct { + src [4]uint32 + dst [4]uint32 + connectionOpened uint32 + connectionClosed uint32 + receivedBytes uint32 + sentBytes uint32 + success bool +} + +type commonTrafficLabels struct { + direction string + + sourceWorkload string + sourceCanonicalService string + sourceCanonicalRevision string + sourceWorkloadNamespace string + sourcePrincipal string + sourceApp string + sourceVersion string + sourceCluster string + + destinationService string + destinationServiceNamespace string + destinationServiceName string + destinationWorkload string + destinationCanonicalService string + destinationCanonicalRevision string + destinationWorkloadNamespace string + destinationPrincipal string + destinationApp string + destinationVersion string + destinationCluster string + + requestProtocol string + responseFlags string + connectionSecurityPolicy string +} + +func NewMetric(workloadCache cache.WorkloadCache) *MetricController { + return &MetricController{ + workloadCache: workloadCache, + } +} + +func (m *MetricController) Run(ctx context.Context, mapOfMetricNotify, mapOfMetric *ebpf.Map) { + if m == nil { + return + } + + reader, err := ringbuf.NewReader(mapOfMetricNotify) + if err != nil { + log.Errorf("open metric notify ringbuf map FAILED, err: %v", err) + return + } + defer func() { + if err := reader.Close(); err != nil { + log.Errorf("ringbuf reader Close FAILED, err: %v", err) + } + }() + + // Register metrics to Prometheus and start Prometheus server + go RunPrometheusClient(ctx) + + rec := ringbuf.Record{} + key := metricKey{} + value := metricValue{} + data := requestMetric{} + + for { + select { + case <-ctx.Done(): + return + default: + if err := reader.ReadInto(&rec); err != nil { + log.Errorf("ringbuf reader FAILED to read, err: %v", err) + continue + } + + buf := bytes.NewBuffer(rec.RawSample) + if err := binary.Read(buf, binary.LittleEndian, &key); err != nil { + log.Error("get metric key FAILED, err:", err) + continue + } + + if err := mapOfMetric.Lookup(&key, &value); err != nil { + log.Error("get bpf map of metric FAILED, err:", err) + continue + } + + data.src = key.SrcIp + data.dst = key.DstIp + data.connectionClosed = value.ConnectionClose + data.connectionOpened = value.ConnectionOpen + data.sentBytes = value.SentBytes + data.receivedBytes = value.ReceivedBytes + data.success = true + + commonTrafficLabels, err := m.buildMetric(&data) + if err != nil { + log.Warnf("reporter records error") + } + + commonTrafficLabels.direction = "-" + if value.Direction == constants.INBOUND { + commonTrafficLabels.direction = "INBOUND" + } + if value.Direction == constants.OUTBOUND { + commonTrafficLabels.direction = "OUTBOUND" + } + + buildMetricsToPrometheus(data, commonTrafficLabels) + } + } +} + +func (m *MetricController) buildMetric(data *requestMetric) (commonTrafficLabels, error) { + var dstAddr, srcAddr []byte + for i := range data.dst { + dstAddr = binary.LittleEndian.AppendUint32(dstAddr, data.dst[i]) + srcAddr = binary.LittleEndian.AppendUint32(srcAddr, data.src[i]) + } + + dstWorkload, dstIP := m.getWorkloadByAddress(restoreIPv4(dstAddr)) + srcWorkload, _ := m.getWorkloadByAddress(restoreIPv4(srcAddr)) + + trafficLabels := buildMetricFromWorkload(dstWorkload, srcWorkload) + trafficLabels.destinationService = dstIP + + trafficLabels.requestProtocol = "tcp" + trafficLabels.responseFlags = "-" + trafficLabels.connectionSecurityPolicy = "mutual_tls" + + return trafficLabels, nil +} + +func (m *MetricController) getWorkloadByAddress(address []byte) (*workloadapi.Workload, string) { + networkAddr := cache.NetworkAddress{} + networkAddr.Address, _ = netip.AddrFromSlice(address) + workload := m.workloadCache.GetWorkloadByAddr(networkAddr) + if workload == nil { + log.Warnf("get worload from ip %v FAILED", address) + return nil, "" + } + return workload, networkAddr.Address.String() +} + +func buildMetricFromWorkload(dstWorkload, srcWorkload *workloadapi.Workload) commonTrafficLabels { + if dstWorkload == nil || srcWorkload == nil { + return commonTrafficLabels{} + } + + trafficLabels := commonTrafficLabels{} + + trafficLabels.destinationServiceNamespace = dstWorkload.Namespace + trafficLabels.destinationServiceName = dstWorkload.Name + trafficLabels.destinationWorkload = dstWorkload.WorkloadName + trafficLabels.destinationCanonicalService = dstWorkload.CanonicalName + trafficLabels.destinationCanonicalRevision = dstWorkload.CanonicalRevision + trafficLabels.destinationWorkloadNamespace = dstWorkload.Namespace + trafficLabels.destinationApp = dstWorkload.CanonicalName + trafficLabels.destinationVersion = dstWorkload.CanonicalRevision + trafficLabels.destinationCluster = dstWorkload.ClusterId + + trafficLabels.sourceWorkload = srcWorkload.WorkloadName + trafficLabels.sourceCanonicalService = srcWorkload.CanonicalName + trafficLabels.sourceCanonicalRevision = srcWorkload.CanonicalRevision + trafficLabels.sourceWorkloadNamespace = srcWorkload.Namespace + trafficLabels.sourceApp = srcWorkload.CanonicalName + trafficLabels.sourceVersion = srcWorkload.CanonicalRevision + trafficLabels.sourceCluster = srcWorkload.ClusterId + + trafficLabels.destinationPrincipal = buildPrincipal(dstWorkload) + trafficLabels.sourcePrincipal = buildPrincipal(srcWorkload) + + return trafficLabels +} + +func buildPrincipal(workload *workloadapi.Workload) string { + if workload.TrustDomain != "" && workload.ServiceAccount != "" && workload.Namespace != "" { + return fmt.Sprintf("spiffe://%s/ns/%s/sa/%s", workload.TrustDomain, workload.Namespace, workload.ServiceAccount) + } + return "-" +} + +func buildMetricsToPrometheus(data requestMetric, labels commonTrafficLabels) { + commonLabels := commonTrafficLabels2map(&labels) + tcpConnectionOpened.With(commonLabels).Set(float64(data.connectionOpened)) + tcpConnectionClosed.With(commonLabels).Set(float64(data.connectionClosed)) + tcpReceivedBytes.With(commonLabels).Set(float64(data.receivedBytes)) + tcpSentBytes.With(commonLabels).Set(float64(data.sentBytes)) +} + +func commonTrafficLabels2map(labels *commonTrafficLabels) map[string]string { + trafficLabelsMap := make(map[string]string) + + val := reflect.ValueOf(labels).Elem() + num := val.NumField() + for i := 0; i < num; i++ { + fieldInfo := val.Type().Field(i) + if val.Field(i).String() == "" { + trafficLabelsMap[labelsMap[fieldInfo.Name]] = "-" + } else { + trafficLabelsMap[labelsMap[fieldInfo.Name]] = val.Field(i).String() + } + } + + return trafficLabelsMap +} + +// Converting IPv4 data reported in IPv6 form to IPv4 +func restoreIPv4(bytes []byte) []byte { + for i := 4; i < 16; i++ { + if bytes[i] != 0 { + return bytes + } + } + + return bytes[:4] +} diff --git a/pkg/controller/telemetry/metric_test.go b/pkg/controller/telemetry/metric_test.go new file mode 100644 index 000000000..d909eb649 --- /dev/null +++ b/pkg/controller/telemetry/metric_test.go @@ -0,0 +1,493 @@ +/* + * Copyright The Kmesh Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package telemetry + +import ( + "context" + "reflect" + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/assert" + + "kmesh.net/kmesh/api/v2/workloadapi" + "kmesh.net/kmesh/pkg/controller/workload/cache" +) + +func TestCommonTrafficLabels2map(t *testing.T) { + type args struct { + labels *commonTrafficLabels + } + tests := []struct { + name string + args args + want map[string]string + }{ + { + name: "normal commonTrafficLabels to map test", + args: args{ + labels: &commonTrafficLabels{ + direction: "INBOUND", + + sourceWorkload: "sleep", + sourceCanonicalService: "sleep", + sourceCanonicalRevision: "latest", + sourceWorkloadNamespace: "ambient-demo", + sourcePrincipal: "spiffe://cluster.local/ns/ambient-demo/sa/sleep", + sourceApp: "sleep", + sourceVersion: "latest", + sourceCluster: "Kubernetes", + destinationService: "tcp-echo.ambient-demo.svc.cluster.local", + destinationServiceNamespace: "ambient-demo", + destinationServiceName: "tcp-echo", + destinationWorkload: "tcp-echo", + destinationCanonicalService: "tcp-echo", + destinationCanonicalRevision: "v1", + destinationWorkloadNamespace: "ambient-demo", + destinationPrincipal: "spiffe://cluster.local/ns/ambient-demo/sa/default", + destinationApp: "tcp-echo", + destinationVersion: "v1", + destinationCluster: "Kubernetes", + requestProtocol: "tcp", + responseFlags: "-", + connectionSecurityPolicy: "mutual_tls", + }, + }, + want: map[string]string{ + "direction": "INBOUND", + "source_workload": "sleep", + "source_canonical_service": "sleep", + "source_canonical_revision": "latest", + "source_workload_namespace": "ambient-demo", + "source_principal": "spiffe://cluster.local/ns/ambient-demo/sa/sleep", + "source_app": "sleep", + "source_version": "latest", + "source_cluster": "Kubernetes", + "destination_service": "tcp-echo.ambient-demo.svc.cluster.local", + "destination_service_namespace": "ambient-demo", + "destination_service_name": "tcp-echo", + "destination_workload": "tcp-echo", + "destination_canonical_service": "tcp-echo", + "destination_canonical_revision": "v1", + "destination_workload_namespace": "ambient-demo", + "destination_principal": "spiffe://cluster.local/ns/ambient-demo/sa/default", + "destination_app": "tcp-echo", + "destination_version": "v1", + "destination_cluster": "Kubernetes", + "request_protocol": "tcp", + "response_flags": "-", + "connection_security_policy": "mutual_tls", + }, + }, + { + name: "empty commonTrafficLabels to map test", + args: args{ + labels: &commonTrafficLabels{}, + }, + want: map[string]string{ + "direction": "-", + "source_workload": "-", + "source_canonical_service": "-", + "source_canonical_revision": "-", + "source_workload_namespace": "-", + "source_principal": "-", + "source_app": "-", + "source_version": "-", + "source_cluster": "-", + "destination_service": "-", + "destination_service_namespace": "-", + "destination_service_name": "-", + "destination_workload": "-", + "destination_canonical_service": "-", + "destination_canonical_revision": "-", + "destination_workload_namespace": "-", + "destination_principal": "-", + "destination_app": "-", + "destination_version": "-", + "destination_cluster": "-", + "request_protocol": "-", + "response_flags": "-", + "connection_security_policy": "-", + }, + }, + { + name: "Only some fields in the commonTrafficLabels have values", + args: args{ + labels: &commonTrafficLabels{ + direction: "OUTBOUND", + sourceWorkload: "sleep", + destinationWorkload: "tcp-echo", + }, + }, + want: map[string]string{ + "direction": "OUTBOUND", + "source_workload": "sleep", + "source_canonical_service": "-", + "source_canonical_revision": "-", + "source_workload_namespace": "-", + "source_principal": "-", + "source_app": "-", + "source_version": "-", + "source_cluster": "-", + "destination_service": "-", + "destination_service_namespace": "-", + "destination_service_name": "-", + "destination_workload": "tcp-echo", + "destination_canonical_service": "-", + "destination_canonical_revision": "-", + "destination_workload_namespace": "-", + "destination_principal": "-", + "destination_app": "-", + "destination_version": "-", + "destination_cluster": "-", + "request_protocol": "-", + "response_flags": "-", + "connection_security_policy": "-", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := commonTrafficLabels2map(tt.args.labels); !reflect.DeepEqual(got, tt.want) { + t.Errorf("commonTrafficLabels2map() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestBuildMetricsToPrometheus(t *testing.T) { + metrics := []*prometheus.GaugeVec{ + tcpConnectionClosed, + tcpConnectionOpened, + tcpReceivedBytes, + tcpSentBytes, + } + + type args struct { + data requestMetric + labels commonTrafficLabels + } + tests := []struct { + name string + args args + want []float64 + }{ + { + name: "test build metrisc to Prometheus", + args: args{ + data: requestMetric{ + src: [4]uint32{183763210, 0, 0, 0}, + dst: [4]uint32{183762951, 0, 0, 0}, + connectionOpened: 0x0000001, + connectionClosed: 0x0000002, + sentBytes: 0x0000003, + receivedBytes: 0x0000004, + success: true, + }, + labels: commonTrafficLabels{ + direction: "INBOUND", + sourceWorkload: "sleep", + sourceCanonicalService: "sleep", + sourceCanonicalRevision: "latest", + sourceWorkloadNamespace: "ambient-demo", + sourcePrincipal: "spiffe://cluster.local/ns/ambient-demo/sa/sleep", + sourceApp: "sleep", + sourceVersion: "latest", + sourceCluster: "Kubernetes", + destinationService: "tcp-echo.ambient-demo.svc.cluster.local", + destinationServiceNamespace: "ambient-demo", + destinationServiceName: "tcp-echo", + destinationWorkload: "tcp-echo", + destinationCanonicalService: "tcp-echo", + destinationCanonicalRevision: "v1", + destinationWorkloadNamespace: "ambient-demo", + destinationPrincipal: "spiffe://cluster.local/ns/ambient-demo/sa/default", + destinationApp: "tcp-echo", + destinationVersion: "v1", + destinationCluster: "Kubernetes", + requestProtocol: "tcp", + responseFlags: "-", + connectionSecurityPolicy: "mutual_tls", + }, + }, + want: []float64{ + 2, + 1, + 4, + 3, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + go RunPrometheusClient(ctx) + buildMetricsToPrometheus(tt.args.data, tt.args.labels) + commonLabels := commonTrafficLabels2map(&tt.args.labels) + for index, metric := range metrics { + if gauge, err := metric.GetMetricWith(commonLabels); err != nil { + t.Errorf("use labels to get %v failed", metric) + } else { + var m dto.Metric + gauge.Write(&m) + value := m.Gauge.Value + assert.Equal(t, tt.want[index], *value) + } + } + cancel() + }) + } +} + +func TestBuildMetricFromWorkload(t *testing.T) { + type args struct { + dstWorkload *workloadapi.Workload + srcWorkload *workloadapi.Workload + } + tests := []struct { + name string + args args + want commonTrafficLabels + }{ + { + name: "normal capability test", + args: args{ + dstWorkload: &workloadapi.Workload{ + Namespace: "kmesh-system", + Name: "kmesh", + WorkloadName: "kmesh-daemon", + CanonicalName: "dstCanonical", + CanonicalRevision: "dstVersion", + ClusterId: "Kubernetes", + TrustDomain: "cluster.local", + ServiceAccount: "default", + }, + srcWorkload: &workloadapi.Workload{ + Namespace: "kmesh-system", + Name: "kmesh", + WorkloadName: "kmesh-daemon", + CanonicalName: "srcCanonical", + CanonicalRevision: "srcVersion", + ClusterId: "Kubernetes", + TrustDomain: "cluster.local", + ServiceAccount: "default", + }, + }, + want: commonTrafficLabels{ + direction: "-", + sourceWorkload: "kmesh-daemon", + sourceCanonicalService: "srcCanonical", + sourceCanonicalRevision: "srcVersion", + sourceWorkloadNamespace: "kmesh-system", + sourcePrincipal: "spiffe://cluster.local/ns/kmesh-system/sa/default", + sourceApp: "srcCanonical", + sourceVersion: "srcVersion", + sourceCluster: "Kubernetes", + destinationService: "-", + destinationServiceNamespace: "kmesh-system", + destinationServiceName: "kmesh", + destinationWorkload: "kmesh-daemon", + destinationCanonicalService: "dstCanonical", + destinationCanonicalRevision: "dstVersion", + destinationWorkloadNamespace: "kmesh-system", + destinationPrincipal: "spiffe://cluster.local/ns/kmesh-system/sa/default", + destinationApp: "dstCanonical", + destinationVersion: "dstVersion", + destinationCluster: "Kubernetes", + requestProtocol: "-", + responseFlags: "-", + connectionSecurityPolicy: "-", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actualLabels := buildMetricFromWorkload(tt.args.dstWorkload, tt.args.srcWorkload) + expectMap := commonTrafficLabels2map(&tt.want) + actualMap := commonTrafficLabels2map(&actualLabels) + assert.Equal(t, expectMap, actualMap) + }) + } +} + +func TestMetricGetWorkloadByAddress(t *testing.T) { + workload := &workloadapi.Workload{ + Name: "ut-workload", + Uid: "123456", + Addresses: [][]byte{ + {192, 168, 224, 22}, + }, + } + type args struct { + address []byte + } + tests := []struct { + name string + args args + want *workloadapi.Workload + }{ + { + name: "normal capability test", + args: args{ + address: []byte{192, 168, 224, 22}, + }, + want: workload, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := MetricController{ + workloadCache: cache.NewWorkloadCache(), + } + m.workloadCache.AddWorkload(workload) + if got, _ := m.getWorkloadByAddress(tt.args.address); !reflect.DeepEqual(got, tt.want) { + t.Errorf("Metric.getWorkloadByAddress() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestMetricBuildMetric(t *testing.T) { + dstWorkload := &workloadapi.Workload{ + Namespace: "kmesh-system", + Name: "kmesh", + WorkloadName: "kmesh-daemon", + CanonicalName: "dstCanonical", + CanonicalRevision: "dstVersion", + ClusterId: "Kubernetes", + TrustDomain: "cluster.local", + ServiceAccount: "default", + Uid: "123456", + Addresses: [][]byte{ + {192, 168, 224, 22}, + }, + } + srcWorkload := &workloadapi.Workload{ + Namespace: "kmesh-system", + Name: "kmesh", + WorkloadName: "kmesh-daemon", + CanonicalName: "srcCanonical", + CanonicalRevision: "srcVersion", + ClusterId: "Kubernetes", + TrustDomain: "cluster.local", + ServiceAccount: "default", + Uid: "654321", + Addresses: [][]byte{ + {10, 19, 25, 31}, + }, + } + type args struct { + data *requestMetric + } + tests := []struct { + name string + args args + want commonTrafficLabels + wantErr bool + }{ + { + name: "normal capability test", + args: args{ + data: &requestMetric{ + src: [4]uint32{521736970, 0, 0, 0}, + dst: [4]uint32{383822016, 0, 0, 0}, + connectionOpened: uint32(16), + connectionClosed: uint32(8), + sentBytes: uint32(156), + receivedBytes: uint32(1024), + success: true, + }, + }, + want: commonTrafficLabels{ + sourceWorkload: "kmesh-daemon", + sourceCanonicalService: "srcCanonical", + sourceCanonicalRevision: "srcVersion", + sourceWorkloadNamespace: "kmesh-system", + sourcePrincipal: "spiffe://cluster.local/ns/kmesh-system/sa/default", + sourceApp: "srcCanonical", + sourceVersion: "srcVersion", + sourceCluster: "Kubernetes", + destinationService: "192.168.224.22", + destinationServiceNamespace: "kmesh-system", + destinationServiceName: "kmesh", + destinationWorkload: "kmesh-daemon", + destinationCanonicalService: "dstCanonical", + destinationCanonicalRevision: "dstVersion", + destinationWorkloadNamespace: "kmesh-system", + destinationPrincipal: "spiffe://cluster.local/ns/kmesh-system/sa/default", + destinationApp: "dstCanonical", + destinationVersion: "dstVersion", + destinationCluster: "Kubernetes", + requestProtocol: "tcp", + responseFlags: "-", + connectionSecurityPolicy: "mutual_tls", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := MetricController{ + workloadCache: cache.NewWorkloadCache(), + } + m.workloadCache.AddWorkload(dstWorkload) + m.workloadCache.AddWorkload(srcWorkload) + got, err := m.buildMetric(tt.args.data) + if (err != nil) != tt.wantErr { + t.Errorf("Metric.buildMetric() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Metric.buildMetric() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestByteToIpByte(t *testing.T) { + type args struct { + bytes []byte + } + tests := []struct { + name string + args args + want []byte + }{ + { + name: "IPv4 data change", + args: args{ + bytes: []byte{71, 0, 244, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + }, + want: []byte{71, 0, 244, 10}, + }, + { + name: "IPv6 data change", + args: args{ + bytes: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, + }, + want: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := restoreIPv4(tt.args.bytes); !reflect.DeepEqual(got, tt.want) { + t.Errorf("restoreIPv4() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controller/telemetry/utils.go b/pkg/controller/telemetry/utils.go new file mode 100644 index 000000000..f50fa83bc --- /dev/null +++ b/pkg/controller/telemetry/utils.go @@ -0,0 +1,136 @@ +/* + * Copyright The Kmesh Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package telemetry + +import ( + "context" + "net/http" + "sync" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + + "kmesh.net/kmesh/pkg/logger" +) + +var ( + log = logger.NewLoggerField("pkg/telemetry") + mu sync.Mutex + + trafficLabels = []string{ + "direction", + "source_workload", + "source_canonical_service", + "source_canonical_revision", + "source_workload_namespace", + "source_principal", + "source_app", + "source_version", + "source_cluster", + "destination_service", + "destination_service_namespace", + "destination_service_name", + "destination_workload", + "destination_canonical_service", + "destination_canonical_revision", + "destination_workload_namespace", + "destination_principal", + "destination_app", + "destination_version", + "destination_cluster", + "request_protocol", + "response_flags", + "connection_security_policy", + } + + labelsMap = map[string]string{ + "direction": "direction", + "sourceWorkload": "source_workload", + "sourceCanonicalService": "source_canonical_service", + "sourceCanonicalRevision": "source_canonical_revision", + "sourceWorkloadNamespace": "source_workload_namespace", + "sourcePrincipal": "source_principal", + "sourceApp": "source_app", + "sourceVersion": "source_version", + "sourceCluster": "source_cluster", + "destinationService": "destination_service", + "destinationServiceNamespace": "destination_service_namespace", + "destinationServiceName": "destination_service_name", + "destinationWorkload": "destination_workload", + "destinationCanonicalService": "destination_canonical_service", + "destinationCanonicalRevision": "destination_canonical_revision", + "destinationWorkloadNamespace": "destination_workload_namespace", + "destinationPrincipal": "destination_principal", + "destinationApp": "destination_app", + "destinationVersion": "destination_version", + "destinationCluster": "destination_cluster", + "requestProtocol": "request_protocol", + "responseFlags": "response_flags", + "connectionSecurityPolicy": "connection_security_policy", + } +) + +var ( + tcpConnectionOpened = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "kmesh_tcp_connections_opened_total", + Help: "The total number of TCP connections opened", + }, trafficLabels) + + tcpConnectionClosed = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "kmesh_tcp_connections_closed_total", + Help: "The total number of TCP connections closed", + }, trafficLabels) + + tcpReceivedBytes = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "kmesh_tcp_received_bytes_total", + Help: "The size of total bytes received during request in case of a TCP connection", + }, trafficLabels) + + tcpSentBytes = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "kmesh_tcp_sent_bytes_total", + Help: "The size of total bytes sent during response in case of a TCP connection", + }, trafficLabels) +) + +func RunPrometheusClient(ctx context.Context) { + registry := prometheus.NewRegistry() + for { + select { + case <-ctx.Done(): + return + default: + runPrometheusClient(registry) + } + } +} + +func runPrometheusClient(registry *prometheus.Registry) { + // ensure not occur matche the same requests as /status/metric panic + mu.Lock() + defer mu.Unlock() + registry.MustRegister(tcpConnectionOpened, tcpConnectionClosed, tcpReceivedBytes, tcpSentBytes) + + http.Handle("/status/metric", promhttp.HandlerFor(registry, promhttp.HandlerOpts{ + Registry: registry, + })) + if err := http.ListenAndServe(":15020", nil); err != nil { + log.Fatalf("start prometheus client port failed: %v", err) + } +} diff --git a/pkg/controller/telemetry/utils_test.go b/pkg/controller/telemetry/utils_test.go new file mode 100644 index 000000000..7b3989a72 --- /dev/null +++ b/pkg/controller/telemetry/utils_test.go @@ -0,0 +1,84 @@ +/* + * Copyright The Kmesh Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package telemetry + +import ( + "context" + "testing" + + "github.com/prometheus/client_golang/prometheus" +) + +func TestRegisterMetrics(t *testing.T) { + registry := prometheus.NewRegistry() + ctx, cancel := context.WithCancel(context.Background()) + go func() { + for { + select { + case <-ctx.Done(): + return + default: + runPrometheusClient(registry) + } + } + }() + + exportMetrics := []*prometheus.GaugeVec{ + tcpConnectionClosed, + tcpConnectionOpened, + tcpReceivedBytes, + tcpSentBytes, + } + + testlabels := map[string]string{ + "direction": "INBOUND", + "source_workload": "sleep", + "source_canonical_service": "sleep", + "source_canonical_revision": "latest", + "source_workload_namespace": "ambient-demo", + "source_principal": "spiffe://cluster.local/ns/ambient-demo/sa/sleep", + "source_app": "sleep", + "source_version": "latest", + "source_cluster": "Kubernetes", + "destination_service": "tcp-echo.ambient-demo.svc.cluster.local", + "destination_service_namespace": "ambient-demo", + "destination_service_name": "tcp-echo", + "destination_workload": "tcp-echo", + "destination_canonical_service": "tcp-echo", + "destination_canonical_revision": "v1", + "destination_workload_namespace": "ambient-demo", + "destination_principal": "spiffe://cluster.local/ns/ambient-demo/sa/default", + "destination_app": "tcp-echo", + "destination_version": "v1", + "destination_cluster": "Kubernetes", + "request_protocol": "tcp", + "response_flags": "-", + "connection_security_policy": "mutual_tls", + } + + tcpConnectionClosed.With(testlabels).Set(2) + tcpConnectionOpened.With(testlabels).Set(4) + tcpReceivedBytes.With(testlabels).Set(12.64) + tcpSentBytes.With(testlabels).Set(11.45) + + for _, metric := range exportMetrics { + if err := prometheus.Register(metric); err != nil { + t.Errorf("metric not register") + } + } + cancel() +} diff --git a/pkg/controller/workload/bpfcache/backend.go b/pkg/controller/workload/bpfcache/backend.go index 2eb853f67..8178dc520 100644 --- a/pkg/controller/workload/bpfcache/backend.go +++ b/pkg/controller/workload/bpfcache/backend.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/workload/bpfcache/endpoint.go b/pkg/controller/workload/bpfcache/endpoint.go index 38c88f6f8..bb0801d8b 100644 --- a/pkg/controller/workload/bpfcache/endpoint.go +++ b/pkg/controller/workload/bpfcache/endpoint.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/workload/bpfcache/factory.go b/pkg/controller/workload/bpfcache/factory.go index 7fa1fe3b2..819f0e50a 100644 --- a/pkg/controller/workload/bpfcache/factory.go +++ b/pkg/controller/workload/bpfcache/factory.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/workload/bpfcache/fake_map.go b/pkg/controller/workload/bpfcache/fake_map.go index 0d1b4248b..d17a45c3e 100644 --- a/pkg/controller/workload/bpfcache/fake_map.go +++ b/pkg/controller/workload/bpfcache/fake_map.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/workload/bpfcache/frontend.go b/pkg/controller/workload/bpfcache/frontend.go index 804afc304..4856d5e7e 100644 --- a/pkg/controller/workload/bpfcache/frontend.go +++ b/pkg/controller/workload/bpfcache/frontend.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/workload/bpfcache/service.go b/pkg/controller/workload/bpfcache/service.go index 833f83752..c7394cde9 100644 --- a/pkg/controller/workload/bpfcache/service.go +++ b/pkg/controller/workload/bpfcache/service.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/workload/cache/service_cache.go b/pkg/controller/workload/cache/service_cache.go index e9398c48c..8ec056aca 100644 --- a/pkg/controller/workload/cache/service_cache.go +++ b/pkg/controller/workload/cache/service_cache.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/workload/cache/workload_cache.go b/pkg/controller/workload/cache/workload_cache.go index 3c0da4b05..680ef8800 100644 --- a/pkg/controller/workload/cache/workload_cache.go +++ b/pkg/controller/workload/cache/workload_cache.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/workload/cache/workload_cache_test.go b/pkg/controller/workload/cache/workload_cache_test.go index 4691320d6..29e90f915 100644 --- a/pkg/controller/workload/cache/workload_cache_test.go +++ b/pkg/controller/workload/cache/workload_cache_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/workload/workload_controller.go b/pkg/controller/workload/workload_controller.go index 83d512a58..62a50ab8a 100644 --- a/pkg/controller/workload/workload_controller.go +++ b/pkg/controller/workload/workload_controller.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,7 @@ import ( "kmesh.net/kmesh/pkg/auth" "kmesh.net/kmesh/pkg/bpf" + "kmesh.net/kmesh/pkg/controller/telemetry" "kmesh.net/kmesh/pkg/logger" ) @@ -35,10 +36,11 @@ const ( var log = logger.NewLoggerField("workload_controller") type Controller struct { - Stream discoveryv3.AggregatedDiscoveryService_DeltaAggregatedResourcesClient - Processor *Processor - Rbac *auth.Rbac - bpfWorkloadObj *bpf.BpfKmeshWorkload + Stream discoveryv3.AggregatedDiscoveryService_DeltaAggregatedResourcesClient + Processor *Processor + Rbac *auth.Rbac + MetricController *telemetry.MetricController + bpfWorkloadObj *bpf.BpfKmeshWorkload } func NewController(bpfWorkload *bpf.BpfKmeshWorkload) *Controller { @@ -47,11 +49,13 @@ func NewController(bpfWorkload *bpf.BpfKmeshWorkload) *Controller { bpfWorkloadObj: bpfWorkload, } c.Rbac = auth.NewRbac(c.Processor.WorkloadCache) + c.MetricController = telemetry.NewMetric(c.Processor.WorkloadCache) return c } func (c *Controller) Run(ctx context.Context) { go c.Rbac.Run(ctx, c.bpfWorkloadObj.SockOps.MapOfTuple, c.bpfWorkloadObj.XdpAuth.MapOfAuth) + go c.MetricController.Run(ctx, c.bpfWorkloadObj.SockConn.MapOfMetricNotify, c.bpfWorkloadObj.SockConn.MapOfMetrics) } func (c *Controller) WorkloadStreamCreateAndSend(client discoveryv3.AggregatedDiscoveryServiceClient, ctx context.Context) error { diff --git a/pkg/controller/workload/workload_controller_test.go b/pkg/controller/workload/workload_controller_test.go index cf501b9b0..a620eb9c1 100644 --- a/pkg/controller/workload/workload_controller_test.go +++ b/pkg/controller/workload/workload_controller_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/workload/workload_hash.go b/pkg/controller/workload/workload_hash.go index dc6d39208..d066a25b6 100644 --- a/pkg/controller/workload/workload_hash.go +++ b/pkg/controller/workload/workload_hash.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,7 +45,6 @@ func NewHashName() *HashName { } // if read failed, initialize with an empty map if err := hashName.readFromPersistFile(); err != nil { - log.Errorf("error reading persist file: %v", err) hashName.numToStr = make(map[uint32]string) } else { hashName.numToStr = make(map[uint32]string, len(hashName.strToNum)) diff --git a/pkg/controller/workload/workload_hash_test.go b/pkg/controller/workload/workload_hash_test.go index 98e56c7dd..b079cfa3b 100644 --- a/pkg/controller/workload/workload_hash_test.go +++ b/pkg/controller/workload/workload_hash_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/workload/workload_processor.go b/pkg/controller/workload/workload_processor.go index 4016d0e13..ddfbcc79c 100644 --- a/pkg/controller/workload/workload_processor.go +++ b/pkg/controller/workload/workload_processor.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/workload/workload_processor_test.go b/pkg/controller/workload/workload_processor_test.go index 54354d2be..0fe2c333a 100644 --- a/pkg/controller/workload/workload_processor_test.go +++ b/pkg/controller/workload/workload_processor_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/xdstest/fake_xdsclient.go b/pkg/controller/xdstest/fake_xdsclient.go index 489338128..a53b08aa4 100644 --- a/pkg/controller/xdstest/fake_xdsclient.go +++ b/pkg/controller/xdstest/fake_xdsclient.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/controller/xdstest/fake_xdsserver.go b/pkg/controller/xdstest/fake_xdsserver.go index 8a73b619f..019248236 100644 --- a/pkg/controller/xdstest/fake_xdsserver.go +++ b/pkg/controller/xdstest/fake_xdsserver.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/dns/dns.go b/pkg/dns/dns.go index b6053e0a8..9630d4d4d 100644 --- a/pkg/dns/dns.go +++ b/pkg/dns/dns.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/dns/dns_test.go b/pkg/dns/dns_test.go index b1d20db6d..13fc23567 100644 --- a/pkg/dns/dns_test.go +++ b/pkg/dns/dns_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 3ca4b9e55..76351d9ef 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/nets/connect.go b/pkg/nets/connect.go index 0d56eb761..d76430c41 100644 --- a/pkg/nets/connect.go +++ b/pkg/nets/connect.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/nets/nets.go b/pkg/nets/nets.go index 417050a66..542be582a 100644 --- a/pkg/nets/nets.go +++ b/pkg/nets/nets.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/nets/nets_test.go b/pkg/nets/nets_test.go index e8071a1bf..0c9a9dc7b 100644 --- a/pkg/nets/nets_test.go +++ b/pkg/nets/nets_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/status/OWNERS b/pkg/status/OWNERS new file mode 100644 index 000000000..c090c0765 --- /dev/null +++ b/pkg/status/OWNERS @@ -0,0 +1,6 @@ +reviewers: +- hzxuzhonghu +- Okabe-Rintarou-0 +approvers: +- hzxuzhonghu +- Okabe-Rintarou-0 diff --git a/pkg/status/api.go b/pkg/status/api.go index 8cbed740a..5f85c07c8 100644 --- a/pkg/status/api.go +++ b/pkg/status/api.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/status/status_server.go b/pkg/status/status_server.go index ffebe6b54..3b7c27f98 100644 --- a/pkg/status/status_server.go +++ b/pkg/status/status_server.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,6 @@ import ( "strconv" "time" - // nolint "github.com/cilium/ebpf" "github.com/sirupsen/logrus" "google.golang.org/protobuf/encoding/protojson" @@ -52,7 +51,6 @@ const ( patternConfigDumpWorkload = configDumpPrefix + "/workload" patternReadyProbe = "/debug/ready" patternLoggers = "/debug/loggers" - patternBpfLogLevel = "/debug/bpfLogLevel/" bpfLoggerName = "bpf" @@ -168,17 +166,42 @@ func (s *Server) loggersHandler(w http.ResponseWriter, r *http.Request) { } } +func (s *Server) getLoggerNames(w http.ResponseWriter, r *http.Request) { + loggerNames := append(logger.GetLoggerNames(), bpfLoggerName) + data, err := json.MarshalIndent(&loggerNames, "", " ") + if err != nil { + log.Errorf("Failed to marshal logger names: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + _, _ = w.Write(data) +} + func (s *Server) getLoggerLevel(w http.ResponseWriter, r *http.Request) { loggerName := r.URL.Query().Get("name") - loggerLevel, err := logger.GetLoggerLevel(loggerName) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - fmt.Fprintf(w, "\t%v\n", err) + if loggerName == "" { + s.getLoggerNames(w, r) return } - loggerInfo := LoggerInfo{ - Name: loggerName, - Level: loggerLevel.String(), + var loggerInfo *LoggerInfo + if loggerName != bpfLoggerName { + loggerLevel, err := logger.GetLoggerLevel(loggerName) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + fmt.Fprintf(w, "\t%v\n", err) + return + } + loggerInfo = &LoggerInfo{ + Name: loggerName, + Level: loggerLevel.String(), + } + } else { + var err error + loggerInfo, err = s.getBpfLogLevel() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } } data, err := json.MarshalIndent(&loggerInfo, "", " ") if err != nil { @@ -288,6 +311,31 @@ func (s *Server) readyProbe(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("OK")) } +func (s *Server) getBpfLogLevel() (*LoggerInfo, error) { + key := uint32(0) + value := uint32(0) + if err := s.bpfLogLevelMap.Lookup(&key, &value); err != nil { + return nil, fmt.Errorf("get log level error: %v", err) + } + + logLevelMap := map[int]string{ + constants.BPF_LOG_ERR: "error", + constants.BPF_LOG_WARN: "warn", + constants.BPF_LOG_INFO: "info", + constants.BPF_LOG_DEBUG: "debug", + } + + loggerLevel, exists := logLevelMap[int(value)] + if !exists { + return nil, fmt.Errorf("unexpected invalid log level: %d", value) + } + + return &LoggerInfo{ + Name: bpfLoggerName, + Level: loggerLevel, + }, nil +} + func (s *Server) setBpfLogLevel(w http.ResponseWriter, levelStr string) { level, err := strconv.Atoi(levelStr) if err != nil { diff --git a/pkg/status/status_server_test.go b/pkg/status/status_server_test.go index f06b28ebf..98b24f359 100644 --- a/pkg/status/status_server_test.go +++ b/pkg/status/status_server_test.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,16 +22,22 @@ import ( "net/http" "net/http/httptest" "net/netip" + "sort" + "strconv" "testing" "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" "istio.io/istio/pilot/test/util" "kmesh.net/kmesh/api/v2/workloadapi" + "kmesh.net/kmesh/daemon/options" + "kmesh.net/kmesh/pkg/constants" "kmesh.net/kmesh/pkg/controller" "kmesh.net/kmesh/pkg/controller/workload" "kmesh.net/kmesh/pkg/controller/workload/cache" "kmesh.net/kmesh/pkg/logger" + "kmesh.net/kmesh/pkg/utils/test" ) func TestServer_getLoggerLevel(t *testing.T) { @@ -49,28 +55,107 @@ func TestServer_getLoggerLevel(t *testing.T) { w := httptest.NewRecorder() server.getLoggerLevel(w, req) - if w.Code != http.StatusOK { - t.Errorf("Expected status code %d, but got %d", http.StatusOK, w.Code) - } + assert.Equal(t, http.StatusOK, w.Code) var loggerInfo LoggerInfo err := json.Unmarshal(w.Body.Bytes(), &loggerInfo) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } + assert.Nil(t, err) expectedLoggerLevel, err := logger.GetLoggerLevel(loggerName) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } + assert.Nil(t, err) - if expectedLoggerLevel.String() != loggerInfo.Level { - t.Errorf("Wrong logger level, expected %s, but got %s", expectedLoggerLevel.String(), loggerInfo.Level) - } + assert.Equal(t, loggerInfo.Level, expectedLoggerLevel.String()) + assert.Equal(t, loggerInfo.Name, loggerName) + } - if loggerName != loggerInfo.Name { - t.Errorf("Wrong logger name, expected %s, but got %s", loggerName, loggerInfo.Name) - } + req := httptest.NewRequest(http.MethodGet, patternLoggers, nil) + w := httptest.NewRecorder() + server.getLoggerLevel(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + expectedLoggerNames := append(logger.GetLoggerNames(), bpfLoggerName) + var actualLoggerNames []string + err := json.Unmarshal(w.Body.Bytes(), &actualLoggerNames) + assert.Nil(t, err) + + sort.Strings(expectedLoggerNames) + sort.Strings(actualLoggerNames) + assert.Equal(t, expectedLoggerNames, actualLoggerNames) +} + +func TestServer_getAndSetBpfLevel(t *testing.T) { + // Test in two modes + configs := []options.BpfConfig{{ + Mode: "ads", + BpfFsPath: "/sys/fs/bpf", + Cgroup2Path: "/mnt/kmesh_cgroup2", + }, { + Mode: "workload", + BpfFsPath: "/sys/fs/bpf", + Cgroup2Path: "/mnt/kmesh_cgroup2", + }} + + testLoggerLevelMap := map[string]int{ + "error": constants.BPF_LOG_ERR, + "warn": constants.BPF_LOG_WARN, + "info": constants.BPF_LOG_INFO, + "debug": constants.BPF_LOG_DEBUG, + } + key := uint32(0) + actualLoggerLevel := uint32(0) + for _, config := range configs { + t.Run(config.Mode, func(t *testing.T) { + cleanup, bpfLoader := test.InitBpfMap(t, config) + defer cleanup() + server := &Server{ + xdsClient: &controller.XdsClient{ + WorkloadController: &workload.Controller{ + Processor: nil, + }, + }, + bpfLogLevelMap: bpfLoader.GetBpfLogLevel(), + } + + setLoggerUrl := patternLoggers + for logLevelStr, logLevelInt := range testLoggerLevelMap { + // We support both string and number + testLoggerLevels := []string{logLevelStr, strconv.FormatInt(int64(logLevelInt), 10)} + expectedLoggerLevel := uint32(logLevelInt) + for _, testLoggerLevel := range testLoggerLevels { + loggerInfo := LoggerInfo{ + Name: bpfLoggerName, + Level: testLoggerLevel, + } + reqBody, _ := json.Marshal(loggerInfo) + req := httptest.NewRequest(http.MethodPost, setLoggerUrl, bytes.NewReader(reqBody)) + w := httptest.NewRecorder() + server.setLoggerLevel(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + server.bpfLogLevelMap.Lookup(&key, &actualLoggerLevel) + assert.Equal(t, expectedLoggerLevel, actualLoggerLevel) + } + } + + // test get bpf log level + getLoggerUrl := patternLoggers + "?name=" + bpfLoggerName + req := httptest.NewRequest(http.MethodGet, getLoggerUrl, nil) + w := httptest.NewRecorder() + server.getLoggerLevel(w, req) + + var ( + actualLoggerInfo LoggerInfo + expectedLoggerInfo *LoggerInfo + ) + err := json.Unmarshal(w.Body.Bytes(), &actualLoggerInfo) + assert.Nil(t, err) + + expectedLoggerInfo, err = server.getBpfLogLevel() + assert.Nil(t, err) + assert.NotNil(t, expectedLoggerInfo) + assert.Equal(t, expectedLoggerInfo.Level, actualLoggerInfo.Level) + assert.Equal(t, expectedLoggerInfo.Name, actualLoggerInfo.Name) + }) } } @@ -104,18 +189,10 @@ func TestServer_setLoggerLevel(t *testing.T) { w := httptest.NewRecorder() server.setLoggerLevel(w, req) - if w.Code != http.StatusOK { - t.Errorf("Expected status code %d, but got %d", http.StatusOK, w.Code) - } - + assert.Equal(t, http.StatusOK, w.Code) actualLoggerLevel, err := logger.GetLoggerLevel(loggerName) - if err != nil { - t.Errorf("Unexpected error: %v", err) - } - - if actualLoggerLevel.String() != loggerInfo.Level { - t.Errorf("Wrong logger level, expected %s, but got %s", loggerInfo.Level, actualLoggerLevel.String()) - } + assert.Nil(t, err) + assert.Equal(t, loggerInfo.Level, actualLoggerLevel.String()) } } } diff --git a/pkg/utils/ebpf.go b/pkg/utils/ebpf.go index 103512684..1b0ea06c0 100644 --- a/pkg/utils/ebpf.go +++ b/pkg/utils/ebpf.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/utils/exec.go b/pkg/utils/exec.go index eddbd1df4..b51142ee2 100644 --- a/pkg/utils/exec.go +++ b/pkg/utils/exec.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/utils/fileop.go b/pkg/utils/fileop.go index e04952c92..7299f21da 100644 --- a/pkg/utils/fileop.go +++ b/pkg/utils/fileop.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/utils/hash/hash.go b/pkg/utils/hash/hash.go index 609a338b6..d725d45f4 100644 --- a/pkg/utils/hash/hash.go +++ b/pkg/utils/hash/hash.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/utils/kubeclient.go b/pkg/utils/kubeclient.go index a094b4a8a..87ec0c630 100644 --- a/pkg/utils/kubeclient.go +++ b/pkg/utils/kubeclient.go @@ -1,5 +1,5 @@ /* - * Copyright 2023 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,9 +12,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * - * Author: bitcoffee - * Create: 2023-11-19 */ package utils diff --git a/pkg/utils/nettools.go b/pkg/utils/nettools.go index 17d24fddf..13c9416a7 100644 --- a/pkg/utils/nettools.go +++ b/pkg/utils/nettools.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/utils/test/bpf_map.go b/pkg/utils/test/bpf_map.go index e82ba621a..5f680f9d6 100644 --- a/pkg/utils/test/bpf_map.go +++ b/pkg/utils/test/bpf_map.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/pkg/version/version.go b/pkg/version/version.go index 1341c24b7..82cfb6a79 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Kmesh Authors. + * Copyright The Kmesh Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -10,8 +10,10 @@ * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * Create: 2024-5-23 + * See the License for the specific language governing permissions and + * limitations under the License. */ + package version import ( diff --git a/samples/fortio/fortio-header.yaml b/samples/fortio/fortio-header.yaml new file mode 100644 index 000000000..0319091c6 --- /dev/null +++ b/samples/fortio/fortio-header.yaml @@ -0,0 +1,20 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: fortio +spec: + hosts: + - fortio + http: + - match: + - headers: + end-user: + exact: jason + route: + - destination: + host: fortio + subset: v2 + - route: + - destination: + host: fortio + subset: v1 \ No newline at end of file diff --git a/samples/fortio/fortio-route.yaml b/samples/fortio/fortio-route.yaml new file mode 100644 index 000000000..0168ff5ec --- /dev/null +++ b/samples/fortio/fortio-route.yaml @@ -0,0 +1,98 @@ +apiVersion: v1 +kind: Service +metadata: + name: fortio +spec: + ports: + - port: 80 + targetPort: 8080 + selector: + app: fortio +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fortio-v1 +spec: + replicas: 1 + selector: + matchLabels: + app: fortio + version: v1 + template: + metadata: + labels: + app: fortio + version: v1 + spec: + containers: + - name: fortio-server + image: fortio/fortio + imagePullPolicy: IfNotPresent + command: ["fortio"] + args: [ + "server", "-http-port", "0.0.0.0:8080", + "-echo-server-default-params", "header=server:1" + ] + ports: + - containerPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fortio-v2 +spec: + replicas: 1 + selector: + matchLabels: + app: fortio + version: v2 + template: + metadata: + labels: + app: fortio + version: v2 + spec: + containers: + - name: fortio-server + image: fortio/fortio + imagePullPolicy: IfNotPresent + command: ["fortio"] + args: [ + "server", "-http-port", "0.0.0.0:8080", + "-echo-server-default-params", "header=server:2" + ] + ports: + - containerPort: 8080 +--- +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: fortio +spec: + hosts: + - fortio + http: + - route: + - destination: + host: fortio + subset: v1 + weight: 90 + - destination: + host: fortio + subset: v2 + weight: 10 +--- +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: fortio +spec: + host: fortio + subsets: + - name: v1 + labels: + version: v1 + - name: v2 + labels: + version: v2 diff --git a/samples/fortio/fortio-v1-10-v2-90.yaml b/samples/fortio/fortio-v1-10-v2-90.yaml new file mode 100644 index 000000000..06a56c002 --- /dev/null +++ b/samples/fortio/fortio-v1-10-v2-90.yaml @@ -0,0 +1,17 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: fortio +spec: + hosts: + - fortio + http: + - route: + - destination: + host: fortio + subset: v1 + weight: 10 + - destination: + host: fortio + subset: v2 + weight: 90 \ No newline at end of file diff --git a/samples/fortio/netuils.yaml b/samples/fortio/netuils.yaml new file mode 100644 index 000000000..f3f93c8a2 --- /dev/null +++ b/samples/fortio/netuils.yaml @@ -0,0 +1,18 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: netutils +spec: + replicas: 1 + selector: + matchLabels: + app: netutils + template: + metadata: + labels: + app: netutils + spec: + containers: + - name: netutils + image: hwchiu/netutils:latest + imagePullPolicy: IfNotPresent \ No newline at end of file diff --git a/test/e2e/baseline_test.go b/test/e2e/baseline_test.go index 24af5b547..60b08fb39 100644 --- a/test/e2e/baseline_test.go +++ b/test/e2e/baseline_test.go @@ -1,6 +1,22 @@ //go:build integ // +build integ +/* + * Copyright 2024 The Kmesh Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package kmesh import ( diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index f4ba89db9..5d79d362a 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -1,6 +1,22 @@ //go:build integ // +build integ +/* + * Copyright 2024 The Kmesh Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at: + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package kmesh import ( diff --git a/test/e2e/run_test.sh b/test/e2e/run_test.sh index 08e0387d7..0249cd630 100755 --- a/test/e2e/run_test.sh +++ b/test/e2e/run_test.sh @@ -12,7 +12,7 @@ DEFAULT_KIND_IMAGE="kindest/node:v1.30.0@sha256:047357ac0cfea04663786a612ba1eaba # support testing multiple istio version in the future. ISTIO_VERSION=1.22.0 -export KMESH_WAYPOINT_IMAGE="ghcr.io/kmesh-net/waypoint-x86:v0.3.0" +export KMESH_WAYPOINT_IMAGE="ghcr.io/kmesh-net/waypoint:latest" ROOT_DIR=$(git rev-parse --show-toplevel)