Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

neonvm: Make spec min/max/use resources required #971

Merged
merged 4 commits into from
Jun 19, 2024
Merged

neonvm: Make spec min/max/use resources required #971

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
aa7d59b
neonvm: Make spec min/max resources required
sharnoff Jun 18, 2024
2b20f8a
fix vm_controller_test.go
sharnoff Jun 18, 2024
841b976
fix exhaustruct lint
sharnoff Jun 18, 2024
10e389f
Merge branch 'main' into neonvm-require-min-max
sharnoff Jun 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions neonvm/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ spec:
guest:
rootDisk:
image: neondatabase/vm-debian:11
cpus: { min: 1, use: 1, max: 1 }
memorySlots: { min: 1, use: 1, max: 1 }
EOF
```

Expand Down
20 changes: 6 additions & 14 deletions neonvm/apis/neonvm/v1/virtualmachine_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,13 +265,9 @@ type SwapInfo struct {
}

type CPUs struct {
// +optional
// +kubebuilder:default:=1
Min *MilliCPU `json:"min"`
// +optional
Max *MilliCPU `json:"max,omitempty"`
// +optional
Use *MilliCPU `json:"use,omitempty"`
Min MilliCPU `json:"min"`
Max MilliCPU `json:"max"`
Use MilliCPU `json:"use"`
}

// MilliCPU is a special type to represent vCPUs * 1000
Expand Down Expand Up @@ -346,19 +342,15 @@ type MemorySlots struct {
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=128
// +kubebuilder:validation:ExclusiveMaximum=false
// +optional
// +kubebuilder:default:=1
Min *int32 `json:"min"`
Min int32 `json:"min"`
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=128
// +kubebuilder:validation:ExclusiveMaximum=false
// +optional
Max *int32 `json:"max,omitempty"`
Max int32 `json:"max"`
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=128
// +kubebuilder:validation:ExclusiveMaximum=false
// +optional
Use *int32 `json:"use,omitempty"`
Use int32 `json:"use"`
}

type RootDisk struct {
Expand Down
114 changes: 37 additions & 77 deletions neonvm/apis/neonvm/v1/virtualmachine_webhook.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,11 @@ import (
"slices"

ctrl "sigs.k8s.io/controller-runtime"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/webhook"

"k8s.io/apimachinery/pkg/runtime"
)

// log is for logging in this package.
var virtualmachinelog = logf.Log.WithName("virtualmachine-resource")

func (r *VirtualMachine) SetupWebhookWithManager(mgr ctrl.Manager) error {
return ctrl.NewWebhookManagedBy(mgr).
For(r).
Expand All @@ -44,29 +40,7 @@ var _ webhook.Defaulter = &VirtualMachine{}

// Default implements webhook.Defaulter so a webhook will be registered for the type
func (r *VirtualMachine) Default() {
// CPU spec defaulter
if r.Spec.Guest.CPUs.Use == nil {
r.Spec.Guest.CPUs.Use = new(MilliCPU)
*r.Spec.Guest.CPUs.Use = *r.Spec.Guest.CPUs.Min
virtualmachinelog.Info("defaulting guest CPU settings", ".spec.guest.cpus.use", *r.Spec.Guest.CPUs.Use, "VirtualMachine", r.Name)
}
if r.Spec.Guest.CPUs.Max == nil {
r.Spec.Guest.CPUs.Max = new(MilliCPU)
*r.Spec.Guest.CPUs.Max = *r.Spec.Guest.CPUs.Min
virtualmachinelog.Info("defaulting guest CPU settings", ".spec.guest.cpus.max", *r.Spec.Guest.CPUs.Max, "VirtualMachine", r.Name)
}

// Memory spec defaulter
if r.Spec.Guest.MemorySlots.Use == nil {
r.Spec.Guest.MemorySlots.Use = new(int32)
*r.Spec.Guest.MemorySlots.Use = *r.Spec.Guest.MemorySlots.Min
virtualmachinelog.Info("defaulting guest memory settings", ".spec.guest.memorySlots.use", *r.Spec.Guest.MemorySlots.Use, "VirtualMachine", r.Name)
}
if r.Spec.Guest.MemorySlots.Max == nil {
r.Spec.Guest.MemorySlots.Max = new(int32)
*r.Spec.Guest.MemorySlots.Max = *r.Spec.Guest.MemorySlots.Min
virtualmachinelog.Info("defaulting guest memory settings", ".spec.guest.memorySlots.max", *r.Spec.Guest.MemorySlots.Max, "VirtualMachine", r.Name)
}
// Nothing to do.
}

//+kubebuilder:webhook:path=/validate-vm-neon-tech-v1-virtualmachine,mutating=false,failurePolicy=fail,sideEffects=None,groups=vm.neon.tech,resources=virtualmachines,verbs=create;update,versions=v1,name=vvirtualmachine.kb.io,admissionReviewVersions=v1
Expand All @@ -76,37 +50,27 @@ var _ webhook.Validator = &VirtualMachine{}
// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
func (r *VirtualMachine) ValidateCreate() error {
// validate .spec.guest.cpus.use and .spec.guest.cpus.max
if r.Spec.Guest.CPUs.Use != nil {
if r.Spec.Guest.CPUs.Max == nil {
return errors.New(".spec.guest.cpus.max must be defined if .spec.guest.cpus.use specified")
}
if *r.Spec.Guest.CPUs.Use < *r.Spec.Guest.CPUs.Min {
return fmt.Errorf(".spec.guest.cpus.use (%v) should be greater than or equal to the .spec.guest.cpus.min (%v)",
r.Spec.Guest.CPUs.Use,
r.Spec.Guest.CPUs.Min)
}
if *r.Spec.Guest.CPUs.Use > *r.Spec.Guest.CPUs.Max {
return fmt.Errorf(".spec.guest.cpus.use (%v) should be less than or equal to the .spec.guest.cpus.max (%v)",
r.Spec.Guest.CPUs.Use,
r.Spec.Guest.CPUs.Max)
}
if r.Spec.Guest.CPUs.Use < r.Spec.Guest.CPUs.Min {
return fmt.Errorf(".spec.guest.cpus.use (%v) should be greater than or equal to the .spec.guest.cpus.min (%v)",
r.Spec.Guest.CPUs.Use,
r.Spec.Guest.CPUs.Min)
}
if r.Spec.Guest.CPUs.Use > r.Spec.Guest.CPUs.Max {
return fmt.Errorf(".spec.guest.cpus.use (%v) should be less than or equal to the .spec.guest.cpus.max (%v)",
r.Spec.Guest.CPUs.Use,
r.Spec.Guest.CPUs.Max)
}

// validate .spec.guest.memorySlots.use and .spec.guest.memorySlots.max
if r.Spec.Guest.MemorySlots.Use != nil {
if r.Spec.Guest.MemorySlots.Max == nil {
return errors.New(".spec.guest.memorySlots.max must be defined if .spec.guest.memorySlots.use specified")
}
if *r.Spec.Guest.MemorySlots.Use < *r.Spec.Guest.MemorySlots.Min {
return fmt.Errorf(".spec.guest.memorySlots.use (%d) should be greater than or equal to the .spec.guest.memorySlots.min (%d)",
*r.Spec.Guest.MemorySlots.Use,
*r.Spec.Guest.MemorySlots.Min)
}
if *r.Spec.Guest.MemorySlots.Use > *r.Spec.Guest.MemorySlots.Max {
return fmt.Errorf(".spec.guest.memorySlots.use (%d) should be less than or equal to the .spec.guest.memorySlots.max (%d)",
*r.Spec.Guest.MemorySlots.Use,
*r.Spec.Guest.MemorySlots.Max)
}
if r.Spec.Guest.MemorySlots.Use < r.Spec.Guest.MemorySlots.Min {
return fmt.Errorf(".spec.guest.memorySlots.use (%d) should be greater than or equal to the .spec.guest.memorySlots.min (%d)",
r.Spec.Guest.MemorySlots.Use,
r.Spec.Guest.MemorySlots.Min)
}
if r.Spec.Guest.MemorySlots.Use > r.Spec.Guest.MemorySlots.Max {
return fmt.Errorf(".spec.guest.memorySlots.use (%d) should be less than or equal to the .spec.guest.memorySlots.max (%d)",
r.Spec.Guest.MemorySlots.Use,
r.Spec.Guest.MemorySlots.Max)
}

// validate .spec.disk names
Expand Down Expand Up @@ -212,31 +176,27 @@ func (r *VirtualMachine) ValidateUpdate(old runtime.Object) error {
}

// validate .spec.guest.cpu.use
if r.Spec.Guest.CPUs.Use != nil {
if *r.Spec.Guest.CPUs.Use < *r.Spec.Guest.CPUs.Min {
return fmt.Errorf(".cpus.use (%v) should be greater than or equal to the .cpus.min (%v)",
r.Spec.Guest.CPUs.Use,
r.Spec.Guest.CPUs.Min)
}
if *r.Spec.Guest.CPUs.Use > *r.Spec.Guest.CPUs.Max {
return fmt.Errorf(".cpus.use (%v) should be less than or equal to the .cpus.max (%v)",
r.Spec.Guest.CPUs.Use,
r.Spec.Guest.CPUs.Max)
}
if r.Spec.Guest.CPUs.Use < r.Spec.Guest.CPUs.Min {
return fmt.Errorf(".cpus.use (%v) should be greater than or equal to the .cpus.min (%v)",
r.Spec.Guest.CPUs.Use,
r.Spec.Guest.CPUs.Min)
}
if r.Spec.Guest.CPUs.Use > r.Spec.Guest.CPUs.Max {
return fmt.Errorf(".cpus.use (%v) should be less than or equal to the .cpus.max (%v)",
r.Spec.Guest.CPUs.Use,
r.Spec.Guest.CPUs.Max)
}

// validate .spec.guest.memorySlots.use
if r.Spec.Guest.MemorySlots.Use != nil {
if *r.Spec.Guest.MemorySlots.Use < *r.Spec.Guest.MemorySlots.Min {
return fmt.Errorf(".memorySlots.use (%d) should be greater than or equal to the .memorySlots.min (%d)",
*r.Spec.Guest.MemorySlots.Use,
*r.Spec.Guest.MemorySlots.Min)
}
if *r.Spec.Guest.MemorySlots.Use > *r.Spec.Guest.MemorySlots.Max {
return fmt.Errorf(".memorySlots.use (%d) should be less than or equal to the .memorySlots.max (%d)",
*r.Spec.Guest.MemorySlots.Use,
*r.Spec.Guest.MemorySlots.Max)
}
if r.Spec.Guest.MemorySlots.Use < r.Spec.Guest.MemorySlots.Min {
return fmt.Errorf(".memorySlots.use (%d) should be greater than or equal to the .memorySlots.min (%d)",
r.Spec.Guest.MemorySlots.Use,
r.Spec.Guest.MemorySlots.Min)
}
if r.Spec.Guest.MemorySlots.Use > r.Spec.Guest.MemorySlots.Max {
return fmt.Errorf(".memorySlots.use (%d) should be less than or equal to the .memorySlots.max (%d)",
r.Spec.Guest.MemorySlots.Use,
r.Spec.Guest.MemorySlots.Max)
}

return nil
Expand Down
38 changes: 4 additions & 34 deletions neonvm/apis/neonvm/v1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 8 additions & 2 deletions neonvm/config/crd/bases/vm.neon.tech_virtualmachines.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2379,7 +2379,6 @@ spec:
type: integer
x-kubernetes-int-or-string: true
min:
default: 1
description: MilliCPU is a special type to represent vCPUs
* 1000 e.g. 2 vCPU is 2000, 0.25 is 250
format: int32
Expand All @@ -2393,6 +2392,10 @@ spec:
pattern: ^[0-9]+((\.[0-9]*)?|m)
type: integer
x-kubernetes-int-or-string: true
required:
- max
- min
- use
type: object
env:
description: List of environment variables to set in the vmstart
Expand Down Expand Up @@ -2427,7 +2430,6 @@ spec:
minimum: 1
type: integer
min:
default: 1
format: int32
maximum: 128
minimum: 1
Expand All @@ -2437,6 +2439,10 @@ spec:
maximum: 128
minimum: 1
type: integer
required:
- max
- min
- use
type: object
ports:
description: List of ports to expose from the container. Cannot
Expand Down
26 changes: 13 additions & 13 deletions neonvm/controllers/vm_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -551,7 +551,7 @@ func (r *VMReconciler) doReconcile(ctx context.Context, vm *vmv1.VirtualMachine)
// compare guest spec and count of plugged

specUseCPU := vm.Spec.Guest.CPUs.Use
scaleCgroupCPU := *specUseCPU != cgroupUsage.VCPUs
scaleCgroupCPU := specUseCPU != cgroupUsage.VCPUs
scaleQemuCPU := specUseCPU.RoundedUp() != pluggedCPU
if scaleCgroupCPU || scaleQemuCPU {
log.Info("VM goes into scaling mode, CPU count needs to be changed",
Expand All @@ -561,7 +561,7 @@ func (r *VMReconciler) doReconcile(ctx context.Context, vm *vmv1.VirtualMachine)
vm.Status.Phase = vmv1.VmScaling
}

memorySizeFromSpec := resource.NewQuantity(int64(*vm.Spec.Guest.MemorySlots.Use)*vm.Spec.Guest.MemorySlotSize.Value(), resource.BinarySI)
memorySizeFromSpec := resource.NewQuantity(int64(vm.Spec.Guest.MemorySlots.Use)*vm.Spec.Guest.MemorySlotSize.Value(), resource.BinarySI)
if !memorySize.Equal(*memorySizeFromSpec) {
log.Info("VM goes into scale mode, need to resize Memory",
"Memory on board", memorySize,
Expand Down Expand Up @@ -699,13 +699,13 @@ func (r *VMReconciler) doReconcile(ctx context.Context, vm *vmv1.VirtualMachine)
r.Recorder.Event(vm, "Normal", "ScaleDown",
fmt.Sprintf("One CPU was unplugged from VM %s",
vm.Name))
} else if *specCPU != cgroupUsage.VCPUs {
log.Info("Update runner pod cgroups", "runner", cgroupUsage.VCPUs, "spec", *specCPU)
if err := setRunnerCgroup(ctx, vm, *specCPU); err != nil {
} else if specCPU != cgroupUsage.VCPUs {
log.Info("Update runner pod cgroups", "runner", cgroupUsage.VCPUs, "spec", specCPU)
if err := setRunnerCgroup(ctx, vm, specCPU); err != nil {
return err
}
reason := "ScaleDown"
if *specCPU > cgroupUsage.VCPUs {
if specCPU > cgroupUsage.VCPUs {
reason = "ScaleUp"
}
r.Recorder.Event(vm, "Normal", reason,
Expand All @@ -717,8 +717,8 @@ func (r *VMReconciler) doReconcile(ctx context.Context, vm *vmv1.VirtualMachine)
}

// do hotplug/unplug Memory
memSlotsMin := *vm.Spec.Guest.MemorySlots.Min
targetSlotCount := int(*vm.Spec.Guest.MemorySlots.Use - memSlotsMin)
memSlotsMin := vm.Spec.Guest.MemorySlots.Min
targetSlotCount := int(vm.Spec.Guest.MemorySlots.Use - memSlotsMin)

realSlots, err := QmpSetMemorySlots(ctx, vm, targetSlotCount, r.Recorder)
if realSlots < 0 {
Expand All @@ -732,9 +732,9 @@ func (r *VMReconciler) doReconcile(ctx context.Context, vm *vmv1.VirtualMachine)
log.Error(err, "Unable to re-fetch VirtualMachine")
return err
}
memorySlotsUseInSpec := *vm.Spec.Guest.MemorySlots.Use
memorySlotsUseInSpec := vm.Spec.Guest.MemorySlots.Use
memoryPluggedSlots := memSlotsMin + int32(realSlots)
*vm.Spec.Guest.MemorySlots.Use = memoryPluggedSlots
vm.Spec.Guest.MemorySlots.Use = memoryPluggedSlots
if err := r.tryUpdateVM(ctx, vm); err != nil {
log.Error(err, "Failed to update .spec.guest.memorySlots.use",
"old value", memorySlotsUseInSpec,
Expand Down Expand Up @@ -1054,9 +1054,9 @@ func updatePodMetadataIfNecessary(ctx context.Context, c client.Client, vm *vmv1
}

func extractVirtualMachineUsageJSON(spec vmv1.VirtualMachineSpec) string {
cpu := *spec.Guest.CPUs.Use
cpu := spec.Guest.CPUs.Use

memorySlots := *spec.Guest.MemorySlots.Use
memorySlots := spec.Guest.MemorySlots.Use

usage := vmv1.VirtualMachineUsage{
CPU: cpu.ToResourceQuantity(),
Expand Down Expand Up @@ -1423,7 +1423,7 @@ func podSpec(vm *vmv1.VirtualMachine, sshSecret *corev1.Secret, config *Reconcil
Command: []string{
"container-mgr",
"-port", strconv.Itoa(int(vm.Spec.RunnerPort)),
"-init-milli-cpu", strconv.Itoa(int(*vm.Spec.Guest.CPUs.Use)),
"-init-milli-cpu", strconv.Itoa(int(vm.Spec.Guest.CPUs.Use)),
},
Env: []corev1.EnvVar{
{
Expand Down
Loading
Loading