From e5d3dbedf4c1bbaed3061675a4f91d79c214467d Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 15 Jan 2025 20:22:22 +0200 Subject: [PATCH 1/4] test: Update kubectl-gather to latest version Update the install instructions to install the latest version on Linux and macOS, and add a note about required version (0.6.0). Signed-off-by: Nir Soffer --- docs/user-quick-start.md | 8 +++++++- test/README.md | 8 ++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/user-quick-start.md b/docs/user-quick-start.md index b7c3f79df..c79011648 100644 --- a/docs/user-quick-start.md +++ b/docs/user-quick-start.md @@ -209,11 +209,17 @@ enough resources: 1. Install the `kubectl-gather` plugin ``` - curl -L -o kubectl-gather https://github.com/nirs/kubectl-gather/releases/download/v0.4.1/kubectl-gather-v0.4.1-linux-amd64 + tag="$(curl -fsSL https://api.github.com/repos/nirs/kubectl-gather/releases/latest | jq -r .tag_name)" + os="$(uname | tr '[:upper:]' '[:lower:]')" + machine="$(uname -m)" + if [ "$machine" = "aarch64" ]; then machine="arm64"; fi + if [ "$machine" = "x86_64" ]; then machine="amd64"; fi + curl -L -o kubectl-gather https://github.com/nirs/kubectl-gather/releases/download/$tag/kubectl-gather-$tag-$os-$machine sudo install kubectl-gather /usr/local/bin rm kubectl-gather ``` + kubectl-gather version 0.6.0 or later is required. For more info see [kubectl-gather](https://github.com/nirs/kubectl-gather) 1. Install `helm` tool - on Fedora you can use: diff --git a/test/README.md b/test/README.md index eaa407aab..af4fe197d 100644 --- a/test/README.md +++ b/test/README.md @@ -118,11 +118,13 @@ environment. 1. Install the `kubectl-gather` plugin ``` - curl -L -o kubectl-gather https://github.com/nirs/kubectl-gather/releases/download/v0.5.1/kubectl-gather-v0.5.1-linux-amd64 + tag="$(curl -fsSL https://api.github.com/repos/nirs/kubectl-gather/releases/latest | jq -r .tag_name)" + curl -L -o kubectl-gather https://github.com/nirs/kubectl-gather/releases/download/$tag/kubectl-gather-$tag-linux-amd64 sudo install kubectl-gather /usr/local/bin rm kubectl-gather ``` + kubectl-gather version 0.6.0 or later is required. For more info see [kubectl-gather](https://github.com/nirs/kubectl-gather) ## Setup on macOS @@ -158,11 +160,13 @@ environment. 1. Install the `kubectl-gather` plugin ``` - curl -L -o kubectl-gather https://github.com/nirs/kubectl-gather/releases/download/v0.5.1/kubectl-gather-v0.5.1-darwin-arm64 + tag="$(curl -fsSL https://api.github.com/repos/nirs/kubectl-gather/releases/latest | jq -r .tag_name)" + curl -L -o kubectl-gather https://github.com/nirs/kubectl-gather/releases/download/$tag/kubectl-gather-$tag-darwin-arm64 sudo install kubectl-gather /usr/local/bin rm kubectl-gather ``` + kubectl-gather version 0.6.0 or later is required. For more info see [kubectl-gather](https://github.com/nirs/kubectl-gather) 1. Install `socket_vmnet` From 9761e54a945485e6d3c75b6f9f41cb8d19abe517 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 15 Jan 2025 21:17:40 +0200 Subject: [PATCH 2/4] drenv: Add zap module The module parses and log JSON lines logged by programs using Go zap module. This is needed for logging kubectl-gather JSON logs, but can also be used with other tools, since zap is very popular. Signed-off-by: Nir Soffer --- test/drenv/zap.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 test/drenv/zap.py diff --git a/test/drenv/zap.py b/test/drenv/zap.py new file mode 100644 index 000000000..980cce94e --- /dev/null +++ b/test/drenv/zap.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: The RamenDR authors +# SPDX-License-Identifier: Apache-2.0 + +import json +import logging + +# Map zap log levels to python log levels. +# https://github.com/uber-go/zap/blob/6d482535bdd97f4d97b2f9573ac308f1cf9b574e/zapcore/level.go#L113 +_LOGGERS = { + "debug": logging.debug, + "info": logging.info, + "warn": logging.warn, + "error": logging.error, + "dpanic": logging.error, + "panic": logging.error, + "fatal": logging.error, +} + + +def log_json(line, name="zap"): + info = json.loads(line) + info.pop("ts", None) + msg = info.pop("msg", None) + level = info.pop("level", None) + logger = info.pop("logger", name) + log = _LOGGERS.get(level.lower(), logging.info) + if info: + log("[%s] %s %s", logger, msg, info) + else: + log("[%s] %s", logger, msg) From f011484906af570dff3ae90b491a3cc29dec5e93 Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Tue, 14 Jan 2025 13:21:01 +0200 Subject: [PATCH 3/4] drenv: Improve gather logs We used to drop gather info logs. Watch the gather process and log the gather logs as they are written to stderr. Example logs with this change: % drenv gather envs/regional-dr.yaml 2025-01-15 22:04:20,865 INFO [rdr] Gathering environment 2025-01-15 22:04:21,124 INFO [gather] Using kubeconfig "/Users/nsoffer/.kube/config" 2025-01-15 22:04:21,126 INFO [gather] Gathering from all namespaces 2025-01-15 22:04:21,126 INFO [gather] Using all addons 2025-01-15 22:04:21,126 INFO [gather] Storing data in "gather.20250115220421" 2025-01-15 22:04:21,126 INFO [gather] Gathering from cluster "dr1" 2025-01-15 22:04:21,126 INFO [gather] Gathering from cluster "dr2" 2025-01-15 22:04:21,126 INFO [gather] Gathering from cluster "hub" 2025-01-15 22:04:23,095 INFO [gather] Gathered 1535 resources from cluster "hub" in 1.969 seconds 2025-01-15 22:04:24,395 INFO [gather] Gathered 1909 resources from cluster "dr1" in 3.269 seconds 2025-01-15 22:04:24,544 INFO [gather] Gathered 1920 resources from cluster "dr2" in 3.417 seconds 2025-01-15 22:04:24,544 INFO [gather] Gathered 5364 resources from 3 clusters in 3.418 seconds 2025-01-15 22:04:24,555 INFO [rdr] Environment gathered in 3.69 seconds Note: This change requires kubectl-gather 0.6.0. Developers that want to use the `drenv gather` command will have to upgrade kubectl-gather to latest version. Signed-off-by: Nir Soffer --- test/drenv/__main__.py | 1 + test/drenv/kubectl.py | 28 ++++++++++++++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/test/drenv/__main__.py b/test/drenv/__main__.py index 7fd709483..039ae8012 100644 --- a/test/drenv/__main__.py +++ b/test/drenv/__main__.py @@ -285,6 +285,7 @@ def do_gather(args): [p["name"] for p in env["profiles"]], directory=args.directory, namespaces=args.namespaces, + name=env["name"], ) logging.info( "[%s] Environment gathered in %.2f seconds", diff --git a/test/drenv/kubectl.py b/test/drenv/kubectl.py index 693129ade..28bfbc9f0 100644 --- a/test/drenv/kubectl.py +++ b/test/drenv/kubectl.py @@ -1,7 +1,11 @@ # SPDX-FileCopyrightText: The RamenDR authors # SPDX-License-Identifier: Apache-2.0 +import logging +import subprocess + from . import commands +from . import zap JSONPATH_NEWLINE = '{"\\n"}' @@ -188,16 +192,32 @@ def watch( return commands.watch(*cmd, timeout=timeout) -def gather(contexts, namespaces=None, directory=None): +def gather(contexts, namespaces=None, directory=None, name="gather"): """ - Run kubectl gather plugin. + Run kubectl gather plugin, logging gather logs. """ - cmd = ["kubectl", "gather", "--contexts", ",".join(contexts)] + cmd = [ + "kubectl", + "gather", + "--log-format", + "json", + "--contexts", + ",".join(contexts), + ] if namespaces: cmd.extend(("--namespaces", ",".join(namespaces))) if directory: cmd.extend(("--directory", directory)) - commands.run(*cmd) + + # Redirecting stderr to stdout to get the logs. kubectl-gather does not + # output anything to stdout. + for line in commands.watch(*cmd, stderr=subprocess.STDOUT): + try: + zap.log_json(line, name=name) + except ValueError: + # We don't want to crash if kubectl-gather has a logging bug, and + # the line may contain useful info. + logging.debug("[%s] %s", name, line) def _run(cmd, *args, env=None, context=None): From 40a627c03b6e4b467c5a67d4c8f0d4645f227dce Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Wed, 15 Jan 2025 21:50:31 +0200 Subject: [PATCH 4/4] drenv: Support verbose gather If drenv is run in verbose mode, pass --verbose flag to kubectl-gather to log more verbose gather logs. This may be useful to debug issues in kubectl gather in the CI environment. Example log with this change: % drenv gather envs/regional-dr.yaml -v 2025-01-15 22:13:31,064 DEBUG [envfile] Detected os: 'darwin' 2025-01-15 22:13:31,064 DEBUG [envfile] Detected machine: 'arm64' 2025-01-15 22:13:31,064 DEBUG [envfile] Using provider: 'lima' 2025-01-15 22:13:31,064 DEBUG [envfile] Using driver: '' 2025-01-15 22:13:31,064 DEBUG [envfile] Using network: '' 2025-01-15 22:13:31,064 DEBUG [envfile] Detected os: 'darwin' 2025-01-15 22:13:31,064 DEBUG [envfile] Detected machine: 'arm64' 2025-01-15 22:13:31,064 DEBUG [envfile] Using provider: 'lima' 2025-01-15 22:13:31,064 DEBUG [envfile] Using driver: '' 2025-01-15 22:13:31,064 DEBUG [envfile] Using network: '' 2025-01-15 22:13:31,064 DEBUG [envfile] Detected os: 'darwin' 2025-01-15 22:13:31,064 DEBUG [envfile] Detected machine: 'arm64' 2025-01-15 22:13:31,064 DEBUG [envfile] Using provider: 'lima' 2025-01-15 22:13:31,064 DEBUG [envfile] Using driver: '' 2025-01-15 22:13:31,064 DEBUG [envfile] Using network: '' 2025-01-15 22:13:31,064 INFO [rdr] Gathering environment 2025-01-15 22:13:31,356 INFO [gather] Using kubeconfig "/Users/nsoffer/.kube/config" 2025-01-15 22:13:31,359 INFO [gather] Gathering from all namespaces 2025-01-15 22:13:31,359 INFO [gather] Using all addons 2025-01-15 22:13:31,359 INFO [gather] Storing data in "gather.20250115221331" 2025-01-15 22:13:31,359 INFO [gather] Gathering from cluster "dr1" 2025-01-15 22:13:31,359 INFO [gather] Gathering from cluster "dr2" 2025-01-15 22:13:31,359 INFO [gather] Gathering from cluster "hub" 2025-01-15 22:13:31,406 DEBUG [gather.dr2] Listed 133 api resources in 0.038 seconds 2025-01-15 22:13:31,406 DEBUG [gather.hub] Listed 105 api resources in 0.038 seconds 2025-01-15 22:13:31,410 DEBUG [gather.dr1] Listed 133 api resources in 0.042 seconds 2025-01-15 22:13:31,416 DEBUG [gather.dr2] Listed 0 "podtemplates" in 0.010 seconds 2025-01-15 22:13:31,416 DEBUG [gather.dr2] Gathered 0 "podtemplates" in 0.010 seconds Signed-off-by: Nir Soffer --- test/drenv/__main__.py | 1 + test/drenv/kubectl.py | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/test/drenv/__main__.py b/test/drenv/__main__.py index 039ae8012..fe36a5d3f 100644 --- a/test/drenv/__main__.py +++ b/test/drenv/__main__.py @@ -286,6 +286,7 @@ def do_gather(args): directory=args.directory, namespaces=args.namespaces, name=env["name"], + verbose=args.verbose, ) logging.info( "[%s] Environment gathered in %.2f seconds", diff --git a/test/drenv/kubectl.py b/test/drenv/kubectl.py index 28bfbc9f0..5f34b158d 100644 --- a/test/drenv/kubectl.py +++ b/test/drenv/kubectl.py @@ -192,7 +192,7 @@ def watch( return commands.watch(*cmd, timeout=timeout) -def gather(contexts, namespaces=None, directory=None, name="gather"): +def gather(contexts, namespaces=None, directory=None, name="gather", verbose=False): """ Run kubectl gather plugin, logging gather logs. """ @@ -208,6 +208,8 @@ def gather(contexts, namespaces=None, directory=None, name="gather"): cmd.extend(("--namespaces", ",".join(namespaces))) if directory: cmd.extend(("--directory", directory)) + if verbose: + cmd.append("--verbose") # Redirecting stderr to stdout to get the logs. kubectl-gather does not # output anything to stdout.