From dceb676527fa1b593a0cba28c1327cdc16681c5f Mon Sep 17 00:00:00 2001 From: Raghavendra Talur Date: Tue, 24 Aug 2021 13:58:58 -0400 Subject: [PATCH] Add extra disks capability to kvm2 driver Having additional disks on the nodes is a requirement for developers working on the storage components in Kubernetes. This commit adds the extra-disks feature to the kvm2 driver. Signed-off-by: Raghavendra Talur --- cmd/minikube/cmd/start_flags.go | 4 +- pkg/drivers/kvm/disks.go | 80 ++++++++++++++++++++++ pkg/drivers/kvm/domain_definition_arm64.go | 3 + pkg/drivers/kvm/domain_definition_x86.go | 3 + pkg/drivers/kvm/kvm.go | 26 +++++++ pkg/minikube/config/types.go | 2 +- pkg/minikube/registry/drvs/kvm2/kvm2.go | 2 + site/content/en/docs/commands/start.md | 2 +- 8 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 pkg/drivers/kvm/disks.go diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index f87fd108e654..80c7edc728f7 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -168,7 +168,7 @@ func initMinikubeFlags() { startCmd.Flags().StringP(network, "", "", "network to run minikube with. Now it is used by docker/podman and KVM drivers. If left empty, minikube will create a new network.") startCmd.Flags().StringVarP(&outputFormat, "output", "o", "text", "Format to print stdout in. Options include: [text,json]") startCmd.Flags().StringP(trace, "", "", "Send trace events. Options include: [gcp]") - startCmd.Flags().Int(extraDisks, 0, "Number of extra disks created and attached to the minikube VM (currently only implemented for hyperkit driver)") + startCmd.Flags().Int(extraDisks, 0, "Number of extra disks created and attached to the minikube VM (currently only implemented for hyperkit and kvm2 drivers)") } // initKubernetesFlags inits the commandline flags for Kubernetes related options @@ -730,7 +730,7 @@ func interpretWaitFlag(cmd cobra.Command) map[string]bool { } func checkExtraDiskOptions(cmd *cobra.Command, driverName string) { - supportedDrivers := []string{driver.HyperKit} + supportedDrivers := []string{driver.HyperKit, driver.KVM2} if cmd.Flags().Changed(extraDisks) { supported := false diff --git a/pkg/drivers/kvm/disks.go b/pkg/drivers/kvm/disks.go new file mode 100644 index 000000000000..10c9bd5025ae --- /dev/null +++ b/pkg/drivers/kvm/disks.go @@ -0,0 +1,80 @@ +// +build linux + +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kvm + +import ( + "bytes" + "fmt" + "os" + "text/template" + + "github.com/docker/machine/libmachine/log" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/drivers" + "k8s.io/minikube/pkg/util" +) + +// extraDisksTmpl ExtraDisks XML Template +const extraDisksTmpl = ` + + + + + +` + +// ExtraDisks holds the extra disks configuration +type ExtraDisks struct { + DiskPath string + DiskLogicalName string +} + +// getExtraDiskXML returns the XML that can be added to the libvirt domain XML +// for additional disks +func getExtraDiskXML(diskpath string, logicalName string) (string, error) { + var extraDisk ExtraDisks + extraDisk.DiskLogicalName = logicalName + extraDisk.DiskPath = diskpath + tmpl := template.Must(template.New("").Parse(extraDisksTmpl)) + var extraDisksXML bytes.Buffer + if err := tmpl.Execute(&extraDisksXML, extraDisk); err != nil { + return "", fmt.Errorf("couldn't generate extra disks XML: %v", err) + } + return extraDisksXML.String(), nil +} + +// createExtraDisks creates the extra disk files +func createExtraDisk(d *Driver, index int) (string, error) { + diskPath := drivers.ExtraDiskPath(d.BaseDriver, index) + log.Infof("Creating raw disk image: %s of size %v", diskPath, d.DiskSize) + + if _, err := os.Stat(diskPath); os.IsNotExist(err) { + file, err := os.OpenFile(diskPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) + if err != nil { + return "", errors.Wrap(err, "open") + } + defer file.Close() + + if err := file.Truncate(util.ConvertMBToBytes(d.DiskSize)); err != nil { + return "", errors.Wrap(err, "truncate") + } + } + return diskPath, nil + +} diff --git a/pkg/drivers/kvm/domain_definition_arm64.go b/pkg/drivers/kvm/domain_definition_arm64.go index 02575813fe2e..721efbf61a61 100644 --- a/pkg/drivers/kvm/domain_definition_arm64.go +++ b/pkg/drivers/kvm/domain_definition_arm64.go @@ -77,6 +77,9 @@ const domainTmpl = ` {{if .GPU}} {{.DevicesXML}} {{end}} + {{if gt .ExtraDisks 0}} + {{.ExtraDisksXML}} + {{end}} ` diff --git a/pkg/drivers/kvm/domain_definition_x86.go b/pkg/drivers/kvm/domain_definition_x86.go index 2e2627e53c90..9e5c35f193f5 100644 --- a/pkg/drivers/kvm/domain_definition_x86.go +++ b/pkg/drivers/kvm/domain_definition_x86.go @@ -75,6 +75,9 @@ const domainTmpl = ` {{if .GPU}} {{.DevicesXML}} {{end}} + {{if gt .ExtraDisks 0}} + {{.ExtraDisksXML}} + {{end}} ` diff --git a/pkg/drivers/kvm/kvm.go b/pkg/drivers/kvm/kvm.go index 87345505ae4b..9d71ff6359c0 100644 --- a/pkg/drivers/kvm/kvm.go +++ b/pkg/drivers/kvm/kvm.go @@ -88,6 +88,12 @@ type Driver struct { // NUMA XML NUMANodeXML string + + // Extra Disks + ExtraDisks int + + // Extra Disks XML + ExtraDisksXML []string } const ( @@ -352,6 +358,26 @@ func (d *Driver) Create() (err error) { return errors.Wrap(err, "error creating disk") } + if d.ExtraDisks > 20 { + // Limiting the number of disks to 20 arbitrarily. If more disks are + // needed, the logical name generation has to changed to create them if + // the form hdaa, hdab, etc + return errors.Wrap(err, "cannot create more than 20 extra disks") + } + for i := 0; i < d.ExtraDisks; i++ { + diskpath, err := createExtraDisk(d, i) + if err != nil { + return errors.Wrap(err, "creating extra disks") + } + // Starting the logical names for the extra disks from hdd as the cdrom device is set to hdc. + // TODO: Enhance the domain template to use variable for the logical name of the main disk and the cdrom disk. + extraDisksXML, err := getExtraDiskXML(diskpath, fmt.Sprintf("hd%v", string(rune('d'+i)))) + if err != nil { + return errors.Wrap(err, "creating extraDisk XML") + } + d.ExtraDisksXML = append(d.ExtraDisksXML, extraDisksXML) + } + if err := ensureDirPermissions(store); err != nil { log.Errorf("unable to ensure permissions on %s: %v", store, err) } diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index e9a858a6c2c5..2fcc3ca69039 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -83,7 +83,7 @@ type ClusterConfig struct { ListenAddress string // Only used by the docker and podman driver Network string // only used by docker driver MultiNodeRequested bool - ExtraDisks int // currently only implemented for hyperkit + ExtraDisks int // currently only implemented for hyperkit and kvm2 } // KubernetesConfig contains the parameters used to configure the VM Kubernetes. diff --git a/pkg/minikube/registry/drvs/kvm2/kvm2.go b/pkg/minikube/registry/drvs/kvm2/kvm2.go index 619873b4d02a..73916e5f4952 100644 --- a/pkg/minikube/registry/drvs/kvm2/kvm2.go +++ b/pkg/minikube/registry/drvs/kvm2/kvm2.go @@ -70,6 +70,7 @@ type kvmDriver struct { Hidden bool ConnectionURI string NUMANodeCount int + ExtraDisks int } func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { @@ -92,6 +93,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { Hidden: cc.KVMHidden, ConnectionURI: cc.KVMQemuURI, NUMANodeCount: cc.KVMNUMACount, + ExtraDisks: cc.ExtraDisks, }, nil } diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index a57cd14c0752..db47d9e0d4a1 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -48,7 +48,7 @@ minikube start [flags] The key should be '.' separated, and the first part before the dot is the component to apply the configuration to. Valid components are: kubelet, kubeadm, apiserver, controller-manager, etcd, proxy, scheduler Valid kubeadm parameters: ignore-preflight-errors, dry-run, kubeconfig, kubeconfig-dir, node-name, cri-socket, experimental-upload-certs, certificate-key, rootfs, skip-phases, pod-network-cidr - --extra-disks int Number of extra disks created and attached to the minikube VM (currently only implemented for hyperkit driver) + --extra-disks int Number of extra disks created and attached to the minikube VM (currently only implemented for hyperkit and kvm2 drivers) --feature-gates string A set of key=value pairs that describe feature gates for alpha/experimental features. --force Force minikube to perform possibly dangerous operations --force-systemd If set, force the container runtime to use systemd as cgroup manager. Defaults to false.