Merge branch 'main-v5' into ej/normalizeLabelFlags

pkg/client/cluster.go

@@ -42,7 +42,6 @@ import (
 	runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors"
 	"github.com/rancher/k3d/v4/pkg/types"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
-	"github.com/rancher/k3d/v4/pkg/types/fixes"
 	"github.com/rancher/k3d/v4/pkg/types/k3s"
 	"github.com/rancher/k3d/v4/pkg/util"
 	"github.com/rancher/k3d/v4/version"
@@ -241,22 +240,6 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf
 		})
 	}
 
-	// FIXME: FixCgroupV2 - to be removed when fixed upstream
-	if fixes.FixCgroupV2Enabled() {
-
-		log.Debugln("experimental cgroupv2 fix enabled")
-
-		clusterConfig.ClusterCreateOpts.NodeHooks = append(clusterConfig.ClusterCreateOpts.NodeHooks, k3d.NodeHook{
-			Stage: k3d.LifecycleStagePreStart,
-			Action: actions.WriteFileAction{
-				Runtime: runtime,
-				Content: fixes.CgroupV2Entrypoint,
-				Dest:    "/bin/entrypoint.sh",
-				Mode:    0744,
-			},
-		})
-	}
-
 	return nil
 
 }

pkg/client/fixes.go

@@ -0,0 +1,53 @@
+/*
+Copyright © 2020 The k3d Author(s)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package client
+
+import (
+	"os"
+	"strconv"
+
+	"github.com/rancher/k3d/v4/pkg/runtimes"
+	"github.com/rancher/k3d/v4/pkg/types/fixes"
+	log "github.com/sirupsen/logrus"
+)
+
+// FIXME: FixCgroupV2 - to be removed when fixed upstream
+func EnableCgroupV2FixIfNeeded(runtime runtimes.Runtime) {
+	if _, isSet := os.LookupEnv(fixes.EnvFixCgroupV2); !isSet {
+		runtimeInfo, err := runtime.Info()
+		if err != nil {
+			log.Warnf("Failed to get runtime information: %+v", err)
+			return
+		}
+		cgroupVersion, err := strconv.Atoi(runtimeInfo.CgroupVersion)
+		if err != nil {
+			log.Debugf("Failed to parse cgroupVersion: %+v", err)
+			return
+		}
+		if cgroupVersion == 2 {
+			log.Debugf("Detected CgroupV2, enabling custom entrypoint (disable by setting %s=false)", fixes.EnvFixCgroupV2)
+			if err := os.Setenv(fixes.EnvFixCgroupV2, "true"); err != nil {
+				log.Errorf("Detected CgroupsV2 but failed to enable k3d's hotfix (try `export %s=true`): %+v", fixes.EnvFixCgroupV2, err)
+			}
+		}
+	}
+}

pkg/client/node.go

@@ -25,17 +25,22 @@ package client
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"reflect"
 	"strings"
 	"time"
 
 	dockerunits "github.com/docker/go-units"
 	"github.com/imdario/mergo"
+	"github.com/rancher/k3d/v4/pkg/actions"
 	"github.com/rancher/k3d/v4/pkg/runtimes"
 	"github.com/rancher/k3d/v4/pkg/runtimes/docker"
+	runtimeErrors "github.com/rancher/k3d/v4/pkg/runtimes/errors"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
+	"github.com/rancher/k3d/v4/pkg/types/fixes"
 	"github.com/rancher/k3d/v4/pkg/util"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sync/errgroup"
@@ -89,6 +94,25 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N
 
 	log.Debugf("Adding node %+v \n>>> to cluster %+v\n>>> based on existing node %+v", node, cluster, chosenNode)
 
+	// fetch registry config
+	registryConfigBytes := []byte{}
+	registryConfigReader, err := runtime.ReadFromNode(ctx, k3d.DefaultRegistriesFilePath, chosenNode)
+	if err != nil {
+		if !errors.Is(err, runtimeErrors.ErrRuntimeFileNotFound) {
+			log.Warnf("Failed to read registry config from node %s: %+v", node.Name, err)
+		}
+	} else {
+		defer registryConfigReader.Close()
+
+		var err error
+		registryConfigBytes, err = ioutil.ReadAll(registryConfigReader)
+		if err != nil {
+			log.Warnf("Failed to read registry config from node %s: %+v", node.Name, err)
+		}
+		registryConfigReader.Close()
+		registryConfigBytes = bytes.Trim(registryConfigBytes[512:], "\x00") // trim control characters, etc.
+	}
+
 	// merge node config of new node into existing node config
 	if err := mergo.MergeWithOverwrite(chosenNode, *node); err != nil {
 		log.Errorln("Failed to merge new node config into existing node config")
@@ -133,11 +157,27 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N
 		}
 	}
 
+	// add node actions
+	if len(registryConfigBytes) != 0 {
+		if createNodeOpts.NodeHooks == nil {
+			createNodeOpts.NodeHooks = []k3d.NodeHook{}
+		}
+		createNodeOpts.NodeHooks = append(createNodeOpts.NodeHooks, k3d.NodeHook{
+			Stage: k3d.LifecycleStagePreStart,
+			Action: actions.WriteFileAction{
+				Runtime: runtime,
+				Content: registryConfigBytes,
+				Dest:    k3d.DefaultRegistriesFilePath,
+				Mode:    0644,
+			},
+		})
+	}
+
 	// clear status fields
 	node.State.Running = false
 	node.State.Status = ""
 
-	if err := NodeRun(ctx, runtime, node, k3d.NodeCreateOpts{}); err != nil {
+	if err := NodeRun(ctx, runtime, node, createNodeOpts); err != nil {
 		return err
 	}
 
@@ -162,7 +202,7 @@ func NodeAddToClusterMulti(ctx context.Context, runtime runtimes.Runtime, nodes
 
 	nodeWaitGroup, ctx := errgroup.WithContext(ctx)
 	for _, node := range nodes {
-		if err := NodeAddToCluster(ctx, runtime, node, cluster, k3d.NodeCreateOpts{}); err != nil {
+		if err := NodeAddToCluster(ctx, runtime, node, cluster, createNodeOpts); err != nil {
 			return err
 		}
 		if createNodeOpts.Wait {
@@ -230,8 +270,9 @@ func NodeRun(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, node
 	}
 
 	if err := NodeStart(ctx, runtime, node, k3d.NodeStartOpts{
-		Wait:    nodeCreateOpts.Wait,
-		Timeout: nodeCreateOpts.Timeout,
+		Wait:      nodeCreateOpts.Wait,
+		Timeout:   nodeCreateOpts.Timeout,
+		NodeHooks: nodeCreateOpts.NodeHooks,
 	}); err != nil {
 		return err
 	}
@@ -248,6 +289,27 @@ func NodeStart(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, no
 		return nil
 	}
 
+	// FIXME: FixCgroupV2 - to be removed when fixed upstream
+	if node.Role == k3d.ServerRole || node.Role == k3d.AgentRole {
+		EnableCgroupV2FixIfNeeded(runtime)
+		if fixes.FixCgroupV2Enabled() {
+
+			if nodeStartOpts.NodeHooks == nil {
+				nodeStartOpts.NodeHooks = []k3d.NodeHook{}
+			}
+
+			nodeStartOpts.NodeHooks = append(nodeStartOpts.NodeHooks, k3d.NodeHook{
+				Stage: k3d.LifecycleStagePreStart,
+				Action: actions.WriteFileAction{
+					Runtime: runtime,
+					Content: fixes.CgroupV2Entrypoint,
+					Dest:    "/bin/entrypoint.sh",
+					Mode:    0744,
+				},
+			})
+		}
+	}
+
 	startTime := time.Now()
 	log.Debugf("Node %s Start Time: %+v", node.Name, startTime)
 
@@ -296,6 +358,8 @@ func NodeStart(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, no
 
 // NodeCreate creates a new containerized k3s node
 func NodeCreate(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, createNodeOpts k3d.NodeCreateOpts) error {
+	// FIXME: FixCgroupV2 - to be removed when fixed upstream
+	EnableCgroupV2FixIfNeeded(runtime)
 	log.Tracef("Creating node from spec\n%+v", node)
 
 	/*

pkg/runtimes/docker/util.go

@@ -26,6 +26,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"strings"
 
@@ -35,6 +36,7 @@ import (
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/pkg/errors"
+	runtimeErrors "github.com/rancher/k3d/v4/pkg/runtimes/errors"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
 	log "github.com/sirupsen/logrus"
 )
@@ -140,6 +142,30 @@ func (d Docker) WriteToNode(ctx context.Context, content []byte, dest string, mo
 	return nil
 }
 
+// ReadFromNode reads from a given filepath inside the node container
+func (d Docker) ReadFromNode(ctx context.Context, path string, node *k3d.Node) (io.ReadCloser, error) {
+	log.Tracef("Reading path %s from node %s...", path, node.Name)
+	nodeContainer, err := getNodeContainer(ctx, node)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to find container for node '%s': %+v", node.Name, err)
+	}
+
+	docker, err := GetDockerClient()
+	if err != nil {
+		return nil, err
+	}
+
+	reader, _, err := docker.CopyFromContainer(ctx, nodeContainer.ID, path)
+	if err != nil {
+		if client.IsErrNotFound(err) {
+			return nil, errors.Wrap(runtimeErrors.ErrRuntimeFileNotFound, err.Error())
+		}
+		return nil, err
+	}
+
+	return reader, err
+}
+
 // GetDockerClient returns a docker client
 func GetDockerClient() (*client.Client, error) {
 	var err error

pkg/runtimes/errors/errors.go

@@ -19,7 +19,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
-package runtimes
+package errors
 
 import "errors"
 
@@ -34,3 +34,6 @@ var (
 	ErrRuntimeNetworkNotExists     = errors.New("network does not exist")
 	ErrRuntimeNetworkMultiSameName = errors.New("multiple networks with same name found")
 )
+
+// Container Filesystem Errors
+var ErrRuntimeFileNotFound = errors.New("file not found")

pkg/runtimes/runtime.go

@@ -71,6 +71,7 @@ type Runtime interface {
 	GetImages(context.Context) ([]string, error)
 	CopyToNode(context.Context, string, string, *k3d.Node) error               // @param context, source, destination, node
 	WriteToNode(context.Context, []byte, string, os.FileMode, *k3d.Node) error // @param context, content, destination, filemode, node
+	ReadFromNode(context.Context, string, *k3d.Node) (io.ReadCloser, error)    // @param context, filepath, node
 	GetHostIP(context.Context, string) (net.IP, error)
 	ConnectNodeToNetwork(context.Context, *k3d.Node, string) error      // @param context, node, network name
 	DisconnectNodeFromNetwork(context.Context, *k3d.Node, string) error // @param context, node, network name