From ccdb6c1cb42dd270bd26a2ca43e3e4f1e0d26d33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20L=C3=BCke?= Date: Mon, 21 Jun 2021 21:02:20 +0200 Subject: [PATCH] racker-sim: use IPMI standard port but different loopback address The IPMI serial console did not work behind the UDP forwarder because it used a different port and the IPMI protocol embedds the used port which then mismatches the expected port on the other side. The same port has to be used on both ipmi_sim and the forwarder on the DHCP interface. This requires to use port 623 on the host if we want a standard port and due to the port clash it requires to use different IP addresses, too. Use different loopback addresses which each can listen on port 623. Since listening on port 623 is disallowed for unprivileged users, set ambient caps to allow the ipmi_sim process to listen on this port. --- racker-sim/README.md | 6 +++++- racker-sim/ipmi-env.sh | 12 +++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/racker-sim/README.md b/racker-sim/README.md index 8857b72..ad449f0 100644 --- a/racker-sim/README.md +++ b/racker-sim/README.md @@ -15,7 +15,7 @@ QEMU_ARGS="" ./ipmi-env.sh create nodes.csv 00:11:22:33:44:00 ./flatcar_producti To access the management node use the opened QEMU VGA console, or `ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 22 core@192.168.254.X` where `X` is the IP address you can see in QEMU with `ip a`, -or `ipmitool -C3 -I lanplus -H localhost -p 9011 -U USER -P PASS sol activate` where you can run `echo ssh-rsa AAA... me@mail.com > .ssh/authorized_keys` to +or `ipmitool -C3 -I lanplus -H 127.0.90.11 -U USER -P PASS sol activate` where you can run `echo ssh-rsa AAA... me@mail.com > .ssh/authorized_keys` to add your SSH pub key. Follow the Racker manual PDF on how to install Racker in the management node (`sudo docker run..` and create the `nodes.csv` file under `/usr/share/oem/` etc). @@ -32,6 +32,10 @@ ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p 22 core@192.1 # Afterwards to provision a cluster run: racker bootstrap ``` +The serial console with IPMI from the internal network (e.g., `ipmi NODE` with Racker) only works when the `kernel_console` variable in `lokocfg.vars` is changed to `kernel_console = ["console=ttyS0,115200n8", "earlyprintk=serial,ttyS0,115200n8"]`. + You can pass the `PUBLIC_BRIDGE_PREFIX` env var to `ipmi-env.sh` to choose another /24 subnet prefix for the public bridge, the last byte will be appended (default `192.168.254`). +The IPMI endpoints can also be reached on the host's loopback interface with the IP address `127.0.90.${ID}1` where ID is the node ID starting from 1 for the management node. + By default no VM windows are created because the `QEMU_ARGS` env var defaults to `-nographic` but you can overwrite it as done above with `QEMU_ARGS=""` to have VM windows pop up (requires X11/Wayland). diff --git a/racker-sim/ipmi-env.sh b/racker-sim/ipmi-env.sh index 279d534..d31d845 100755 --- a/racker-sim/ipmi-env.sh +++ b/racker-sim/ipmi-env.sh @@ -20,6 +20,7 @@ if [ $# -lt 1 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then exit 1 fi +/bin/which capsh &> /dev/null || { echo "capsh not found: Install the cpash binary from your distribution" > /dev/stderr ; exit 1 ; } /bin/which ipmi_sim &> /dev/null || { echo "ipmi_sim not found: Install the ipmi_sim binary from your distribution" > /dev/stderr ; exit 1 ; } /bin/which socat &> /dev/null || { echo "socat not found: Install the socat binary from your distribution" > /dev/stderr ; exit 1 ; } /bin/which qemu-system-x86_64 &> /dev/null || { echo "qemu-system-x86_64 not found: Install the qemu-system-x86_64 binary from your distribution" > /dev/stderr ; exit 1 ; } @@ -149,7 +150,7 @@ function config_file() { cat << EOF set_working_mc ${ADDR} startlan 1 -addr :: 90${ID}1 +addr 127.0.90.${ID}1 623 priv_limit admin allowed_auths_callback none md2 md5 straight allowed_auths_user none md2 md5 straight @@ -228,7 +229,7 @@ function create_sim() { running="/proc/$$/fd/${running_fd}" ( set +e - sudo unshare --mount-proc -n -R "${DISK_FOLDER}/node${ID}-bmc" sh -c "ip link set dev lo up; nsenter -a -t 1 ip link set node${ID}bmc0 netns \$\$; ip link set dev node${ID}bmc0 up; dhclient -d --no-pid & socat -T10 udp4-listen:623,reuseaddr,reuseport,fork exec:'nsenter -a -t 1 socat -T10 STDIO udp4\:127.0.0.1\:90${ID}1' & while [ -e '${running}' ]; do sleep 1; done; kill 0; exit 0" & + sudo unshare --mount-proc -n -R "${DISK_FOLDER}/node${ID}-bmc" sh -c "ip link set dev lo up; nsenter -a -t 1 ip link set node${ID}bmc0 netns \$\$; ip link set dev node${ID}bmc0 up; dhclient -d --no-pid & socat -T10 udp4-listen:623,reuseaddr,reuseport,fork exec:'nsenter -a -t 1 socat -T10 STDIO udp4\:127.0.90.${ID}1\:623' & while [ -e '${running}' ]; do sleep 1; done; kill 0; exit 0" & ) sudo ip tuntap add "${TAP0}" mode tap sudo ip link set dev "${TAP0}" up @@ -301,9 +302,10 @@ if [ "$1" = create ]; then create_sim echo "Press Ctrl-C to quit" - config_file > /dev/stderr - command_file > /dev/stderr - ipmi_sim -d --config-file <(config_file) -f <(command_file) --nopersist -n + config_file > /tmp/ipmi-sim/config_file + command_file > /tmp/ipmi-sim/command_file + # Allow the ipmi_sim process to bind to 623 because IPMI embedds the port into the protocol and with UDP forwarding from a different port it complains that the used port mismatches when trying to use the serial console + sudo -E capsh --caps='cap_net_bind_service+eip cap_setpcap,cap_setuid,cap_setgid+ep' --keep=1 --user="$USER" --addamb=cap_net_bind_service -- -c 'exec ipmi_sim -d --config-file /tmp/ipmi-sim/config_file -f /tmp/ipmi-sim/command_file --nopersist -n' cancel else echo "Unknown argument: $@"