diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c9e22f1831..45703fc546 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -427,6 +427,19 @@ rules: - patch - update - watch +- apiGroups: + - nsx.vmware.com + resources: + - subnetsets + - subnetsets/status + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - vmware.infrastructure.cluster.x-k8s.io resources: diff --git a/controllers/vmware/vspherecluster_reconciler.go b/controllers/vmware/vspherecluster_reconciler.go index 23ae472f9b..335c01b8f1 100644 --- a/controllers/vmware/vspherecluster_reconciler.go +++ b/controllers/vmware/vspherecluster_reconciler.go @@ -62,6 +62,7 @@ type ClusterReconciler struct { // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclusters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=vmware.infrastructure.cluster.x-k8s.io,resources=vsphereclustertemplates,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=nsx.vmware.com,resources=subnetsets;subnetsets/status,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=vmware.com,resources=virtualnetworks;virtualnetworks/status,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachinesetresourcepolicies;virtualmachinesetresourcepolicies/status,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=vmoperator.vmware.com,resources=virtualmachineservices;virtualmachineservices/status,verbs=get;list;watch;create;update;patch;delete diff --git a/controllers/vspheremachine_controller.go b/controllers/vspheremachine_controller.go index 8faa2f5626..21901ef123 100644 --- a/controllers/vspheremachine_controller.go +++ b/controllers/vspheremachine_controller.go @@ -90,12 +90,12 @@ func AddMachineControllerToManager(ctx context.Context, controllerManagerContext } if supervisorBased { - r.VMService = &vmoperator.VmopMachineService{Client: controllerManagerContext.Client} networkProvider, err := inframanager.GetNetworkProvider(ctx, controllerManagerContext.Client, controllerManagerContext.NetworkProvider) if err != nil { return errors.Wrap(err, "failed to create a network provider") } r.networkProvider = networkProvider + r.VMService = &vmoperator.VmopMachineService{Client: controllerManagerContext.Client, NetworkProvider: r.networkProvider} return ctrl.NewControllerManagedBy(mgr). // Watch the controlled, infrastructure resource. diff --git a/go.mod b/go.mod index f3813865e0..12e967a31c 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.9.0 github.com/vmware-tanzu/net-operator-api v0.0.0-20231019160108-42131d6e8360 + github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0 github.com/vmware-tanzu/vm-operator/api v1.8.5 github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20231214185006-5477585eebfd github.com/vmware-tanzu/vm-operator/external/tanzu-topology v0.0.0-20231214185006-5477585eebfd diff --git a/go.sum b/go.sum index 9716421b57..824318fa98 100644 --- a/go.sum +++ b/go.sum @@ -550,6 +550,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vmware-tanzu/net-operator-api v0.0.0-20231019160108-42131d6e8360 h1:yG158jviUd3wRqCTJcSDzp+prUZWtSA9dhfm/Rf8m9M= github.com/vmware-tanzu/net-operator-api v0.0.0-20231019160108-42131d6e8360/go.mod h1:dtVG693FvGuOSxJvTaKRVGU0EJR8yvLG3E2VaDDHILM= +github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0 h1:HdnQb/X9vJ8a5WQ03g/0nDr9igIIK1fF6wO5wOtkJT4= +github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk= github.com/vmware-tanzu/vm-operator/api v1.8.5 h1:E8rpRdV8+cNp/eNZ/QUHvlrbpPh8uk6bKqwEEmGWe64= github.com/vmware-tanzu/vm-operator/api v1.8.5/go.mod h1:SXaSFtnw2502Tzy0bfQVHrvbFDijR96r1ihUYQWPOK8= github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20231214185006-5477585eebfd h1:qdfVf7KFW+XX7+D4xC/mlBpRA9+B+opdDPxGdqjxO+4= diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index a42772cc3e..bf953ca51e 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -22,6 +22,7 @@ import ( "github.com/pkg/errors" netopv1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1" + nsxoperatorv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" topologyv1 "github.com/vmware-tanzu/vm-operator/external/tanzu-topology/api/v1alpha1" @@ -64,6 +65,7 @@ func New(ctx context.Context, opts Options) (Manager, error) { _ = vmoprv1.AddToScheme(opts.Scheme) _ = ncpv1.AddToScheme(opts.Scheme) _ = netopv1.AddToScheme(opts.Scheme) + _ = nsxoperatorv1.AddToScheme(opts.Scheme) _ = topologyv1.AddToScheme(opts.Scheme) _ = ipamv1.AddToScheme(opts.Scheme) diff --git a/pkg/manager/network.go b/pkg/manager/network.go index b7f826889b..52850d5f6f 100644 --- a/pkg/manager/network.go +++ b/pkg/manager/network.go @@ -27,6 +27,8 @@ import ( ) const ( + // NSXVPCNetworkProvider identifies the nsx-vpc network provider. + NSXVPCNetworkProvider = "nsx-vpc" // NSXNetworkProvider identifies the NSX network provider. NSXNetworkProvider = "NSX" // VDSNetworkProvider identifies the VDS network provider. @@ -41,6 +43,9 @@ func GetNetworkProvider(ctx context.Context, client client.Client, networkProvid log := ctrl.LoggerFrom(ctx) switch networkProvider { + case NSXVPCNetworkProvider: + log.Info("Pick nsx-vpc network provider") + return network.NsxVpcNetworkProvider(client), nil case NSXNetworkProvider: // TODO: disableFirewall not configurable log.Info("Pick NSX-T network provider") diff --git a/pkg/services/interfaces.go b/pkg/services/interfaces.go index 6404ee398d..68a97fd334 100644 --- a/pkg/services/interfaces.go +++ b/pkg/services/interfaces.go @@ -71,6 +71,9 @@ type NetworkProvider interface { // HasLoadBalancer indicates whether this provider has a load balancer for Services. HasLoadBalancer() bool + // SupportVMReadinessProbe indicates whether this provider support vm readiness probe. + SupportVMReadinessProbe() bool + // ProvisionClusterNetwork creates network resource for a given cluster // This operation should be idempotent ProvisionClusterNetwork(ctx context.Context, clusterCtx *vmware.ClusterContext) error diff --git a/pkg/services/network/constants.go b/pkg/services/network/constants.go index 94c981cd63..360f3af81a 100644 --- a/pkg/services/network/constants.go +++ b/pkg/services/network/constants.go @@ -22,6 +22,10 @@ const ( NSXTTypeNetwork = "nsx-t" // NSXTVNetSelectorKey is also defined in VM Operator. NSXTVNetSelectorKey = "ncp.vmware.com/virtual-network-name" + // NSXVPCTypeNetwork is the name of the NSX VPC network type. + NSXVPCTypeNetwork = "nsx-t-subnetset" + // NSXVPCSubnetSetSelectorKey has to be aligned with VM Operator design. + NSXVPCSubnetSetSelectorKey = "nsx-operator.vmware.com/subnetset-name" // CAPVDefaultNetworkLabel is a label used to identify the default network. CAPVDefaultNetworkLabel = "capv.vmware.com/is-default-network" diff --git a/pkg/services/network/dummy_provider.go b/pkg/services/network/dummy_provider.go index a0c9ce4f01..7f05da2856 100644 --- a/pkg/services/network/dummy_provider.go +++ b/pkg/services/network/dummy_provider.go @@ -38,6 +38,10 @@ func (np *dummyNetworkProvider) HasLoadBalancer() bool { return false } +func (np *dummyNetworkProvider) SupportVMReadinessProbe() bool { + return true +} + func (np *dummyNetworkProvider) ProvisionClusterNetwork(_ context.Context, _ *vmware.ClusterContext) error { return nil } diff --git a/pkg/services/network/netop_provider.go b/pkg/services/network/netop_provider.go index 52511c7222..f4e372c5b7 100644 --- a/pkg/services/network/netop_provider.go +++ b/pkg/services/network/netop_provider.go @@ -48,6 +48,10 @@ func (np *netopNetworkProvider) HasLoadBalancer() bool { return true } +func (np *netopNetworkProvider) SupportVMReadinessProbe() bool { + return true +} + // ProvisionClusterNetwork marks the ClusterNetworkReadyCondition true. func (np *netopNetworkProvider) ProvisionClusterNetwork(_ context.Context, clusterCtx *vmware.ClusterContext) error { conditions.MarkTrue(clusterCtx.VSphereCluster, vmwarev1.ClusterNetworkReadyCondition) diff --git a/pkg/services/network/network_test.go b/pkg/services/network/network_test.go index 049cb85814..abbb203fba 100644 --- a/pkg/services/network/network_test.go +++ b/pkg/services/network/network_test.go @@ -23,6 +23,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" netopv1alpha1 "github.com/vmware-tanzu/net-operator-api/api/v1alpha1" + nsxoperatorv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" ncpv1 "github.com/vmware-tanzu/vm-operator/external/ncp/api/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -42,8 +43,8 @@ import ( const ( // Mocked virtualnetwork status reason and message. - testVnetNotRealizedReason = "Cannot realize network" - testVnetNotRealizedMessage = "NetworkNotRealized" + testNetworkNotRealizedReason = "Cannot realize network" + testNetworkNotRealizedMessage = "NetworkNotRealized" ) func createUnReadyNsxtVirtualNetwork(ctx *vmware.ClusterContext, status ncpv1.VirtualNetworkStatus) *ncpv1.VirtualNetwork { @@ -195,6 +196,34 @@ var _ = Describe("Network provider", func() { Expect(vm.Spec.NetworkInterfaces[0].NetworkType).To(Equal("nsx-t")) }) }) + + Context("with nsxvpc network provider", func() { + BeforeEach(func() { + scheme := runtime.NewScheme() + Expect(ncpv1.AddToScheme(scheme)).To(Succeed()) + client := fake.NewClientBuilder().WithScheme(scheme).Build() + np = NsxVpcNetworkProvider(client) + }) + + It("should add nsxvpc type network interface", func() { + err = np.ConfigureVirtualMachine(ctx, clusterCtx, vm) + Expect(err).ToNot(HaveOccurred()) + Expect(vm.Spec.NetworkInterfaces).To(HaveLen(1)) + Expect(vm.Spec.NetworkInterfaces[0].NetworkType).To(Equal("nsx-t-subnetset")) + }) + + It("nsxvpc network interface already exists", func() { + err = np.ConfigureVirtualMachine(ctx, clusterCtx, vm) + Expect(err).ToNot(HaveOccurred()) + Expect(vm.Spec.NetworkInterfaces).To(HaveLen(1)) + }) + + AfterEach(func() { + Expect(err).ToNot(HaveOccurred()) + Expect(vm.Spec.NetworkInterfaces[0].NetworkName).To(Equal(GetNSXVPCSubnetSetName(vSphereCluster.Name))) + Expect(vm.Spec.NetworkInterfaces[0].NetworkType).To(Equal("nsx-t-subnetset")) + }) + }) }) Context("ProvisionClusterNetwork", func() { @@ -202,6 +231,7 @@ var _ = Describe("Network provider", func() { scheme *runtime.Scheme client runtimeclient.Client nsxNp *nsxtNetworkProvider + vpcNp *nsxvpcNetworkProvider runtimeObjs []runtime.Object vnetObj runtime.Object configmapObj runtime.Object @@ -257,6 +287,7 @@ var _ = Describe("Network provider", func() { Expect(ncpv1.AddToScheme(scheme)).To(Succeed()) Expect(corev1.AddToScheme(scheme)).To(Succeed()) Expect(vmwarev1.AddToScheme(scheme)).To(Succeed()) + Expect(nsxoperatorv1.AddToScheme(scheme)).To(Succeed()) }) Context("with dummy network provider", func() { @@ -459,7 +490,7 @@ var _ = Describe("Network provider", func() { By("create a cluster with virtual network in not ready status") status := ncpv1.VirtualNetworkStatus{ Conditions: []ncpv1.VirtualNetworkCondition{ - {Type: "Ready", Status: "False", Reason: testVnetNotRealizedReason, Message: testVnetNotRealizedMessage}, + {Type: "Ready", Status: "False", Reason: testNetworkNotRealizedReason, Message: testNetworkNotRealizedMessage}, }, } vnetObj = createUnReadyNsxtVirtualNetwork(clusterCtx, status) @@ -470,7 +501,124 @@ var _ = Describe("Network provider", func() { err = np.VerifyNetworkStatus(ctx, clusterCtx, vnetObj) expectedErrorMessage := fmt.Sprintf("virtual network ready status is: '%s' in cluster %s. reason: %s, message: %s", - "False", apitypes.NamespacedName{Namespace: dummyNs, Name: dummyCluster}, testVnetNotRealizedReason, testVnetNotRealizedMessage) + "False", apitypes.NamespacedName{Namespace: dummyNs, Name: dummyCluster}, testNetworkNotRealizedReason, testNetworkNotRealizedMessage) + Expect(err).To(MatchError(expectedErrorMessage)) + Expect(conditions.IsFalse(clusterCtx.VSphereCluster, vmwarev1.ClusterNetworkReadyCondition)).To(BeTrue()) + }) + }) + + Context("with nsxvpc network provider and subnetset exists", func() { + BeforeEach(func() { + client = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(configmapObj, systemNamespaceObj).Build() + vpcNp, _ = NsxVpcNetworkProvider(client).(*nsxvpcNetworkProvider) + np = vpcNp + err = np.ProvisionClusterNetwork(ctx, clusterCtx) + }) + + It("should not update subnetset", func() { + var initialSubnetSetSpec nsxoperatorv1.SubnetSetSpec + + // Fetch the SubnetSet before the operation + initialSubnetSet := &nsxoperatorv1.SubnetSet{} + err = client.Get(ctx, apitypes.NamespacedName{ + Name: GetNSXVPCSubnetSetName(dummyCluster), + Namespace: dummyNs, + }, initialSubnetSet) + Expect(err).NotTo(HaveOccurred()) + initialSubnetSetSpec = initialSubnetSet.Spec + + // Presumably there's code here that might modify the SubnetSet... + Expect(err).ToNot(HaveOccurred()) + subnetset, localerr := np.GetClusterNetworkName(ctx, clusterCtx) + Expect(localerr).ToNot(HaveOccurred()) + Expect(subnetset).To(Equal(GetNSXVPCSubnetSetName(clusterCtx.VSphereCluster.Name))) + + createdSubnetSet := &nsxoperatorv1.SubnetSet{} + err = client.Get(ctx, apitypes.NamespacedName{ + Name: GetNSXVPCSubnetSetName(dummyCluster), + Namespace: dummyNs, + }, createdSubnetSet) + + Expect(err).ToNot(HaveOccurred()) + // Check that the SubnetSetSpec was not changed + Expect(createdSubnetSet.Spec).To(Equal(initialSubnetSetSpec), "SubnetSetSpec should not have been modified") + }) + + It("GetVMServiceAnnotations", func() { + annotations, err := np.GetVMServiceAnnotations(ctx, clusterCtx) + Expect(err).ToNot(HaveOccurred()) + Expect(annotations).To(HaveKeyWithValue(NSXVPCSubnetSetSelectorKey, GetNSXVPCSubnetSetName(clusterCtx.VSphereCluster.Name))) + Expect(conditions.IsTrue(clusterCtx.VSphereCluster, vmwarev1.ClusterNetworkReadyCondition)).To(BeTrue()) + }) + }) + + Context("with nsxvpc network provider and subnetset does not exist", func() { + var nsxvpcNp *nsxvpcNetworkProvider + + BeforeEach(func() { + client = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(configmapObj, systemNamespaceObj).Build() + nsxvpcNp, _ = NsxVpcNetworkProvider(client).(*nsxvpcNetworkProvider) + np = nsxvpcNp + err = np.ProvisionClusterNetwork(ctx, clusterCtx) + }) + + It("should create subnetset with new spec", func() { + Expect(err).ToNot(HaveOccurred()) + subnetset, localerr := np.GetClusterNetworkName(ctx, clusterCtx) + Expect(localerr).ToNot(HaveOccurred()) + Expect(subnetset).To(Equal(GetNSXVPCSubnetSetName(clusterCtx.VSphereCluster.Name))) + + createdSubnetSet := &nsxoperatorv1.SubnetSet{} + err = client.Get(ctx, apitypes.NamespacedName{ + Name: GetNSXVPCSubnetSetName(dummyCluster), + Namespace: dummyNs, + }, createdSubnetSet) + + Expect(err).ToNot(HaveOccurred()) + Expect(createdSubnetSet.Spec.AdvancedConfig.StaticIPAllocation.Enable).To(BeTrue()) + }) + }) + + Context("with nsxvpc network provider failure", func() { + var ( + client runtimeclient.Client + nsxvpcNp *nsxvpcNetworkProvider + scheme *runtime.Scheme + subnetsetObj runtime.Object + ) + + BeforeEach(func() { + scheme = runtime.NewScheme() + Expect(nsxoperatorv1.AddToScheme(scheme)).To(Succeed()) + }) + + It("should return error when subnetset ready status is false", func() { + By("create a cluster with subnetset in not ready status") + status := nsxoperatorv1.SubnetSetStatus{ + Conditions: []nsxoperatorv1.Condition{ + { + Type: "Ready", + Status: "False", + Reason: testNetworkNotRealizedReason, + Message: testNetworkNotRealizedMessage, + }, + }, + } + subnetsetObj = &nsxoperatorv1.SubnetSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: GetNSXVPCSubnetSetName(cluster.Name), + }, + Status: status, + } + client = fake.NewClientBuilder().WithScheme(scheme).WithRuntimeObjects(subnetsetObj).Build() + nsxvpcNp, _ = NsxVpcNetworkProvider(client).(*nsxvpcNetworkProvider) + np = nsxvpcNp + + err = np.VerifyNetworkStatus(ctx, clusterCtx, subnetsetObj) + + expectedErrorMessage := fmt.Sprintf("subnetset ready status is: '%s' in cluster %s. reason: %s, message: %s", + "False", apitypes.NamespacedName{Namespace: dummyNs, Name: dummyCluster}, testNetworkNotRealizedReason, testNetworkNotRealizedMessage) Expect(err).To(MatchError(expectedErrorMessage)) Expect(conditions.IsFalse(clusterCtx.VSphereCluster, vmwarev1.ClusterNetworkReadyCondition)).To(BeTrue()) }) diff --git a/pkg/services/network/nsxt_provider.go b/pkg/services/network/nsxt_provider.go index d6f5cfc096..375fad07ea 100644 --- a/pkg/services/network/nsxt_provider.go +++ b/pkg/services/network/nsxt_provider.go @@ -56,6 +56,10 @@ func (np *nsxtNetworkProvider) HasLoadBalancer() bool { return true } +func (np *nsxtNetworkProvider) SupportVMReadinessProbe() bool { + return true +} + // GetNSXTVirtualNetworkName returns the name of the NSX-T vnet object. func GetNSXTVirtualNetworkName(clusterName string) string { return fmt.Sprintf("%s-vnet", clusterName) diff --git a/pkg/services/network/nsxvpc_provider.go b/pkg/services/network/nsxvpc_provider.go new file mode 100644 index 0000000000..9c130b4e04 --- /dev/null +++ b/pkg/services/network/nsxvpc_provider.go @@ -0,0 +1,178 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package network + +import ( + "context" + "fmt" + + "github.com/pkg/errors" + nsxoperatorv1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" + vmoprv1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util/conditions" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" + "sigs.k8s.io/cluster-api-provider-vsphere/pkg/context/vmware" + "sigs.k8s.io/cluster-api-provider-vsphere/pkg/services" +) + +// nsxtNetworkProvider provision nsx-t type cluster network. +type nsxvpcNetworkProvider struct { + client client.Client +} + +// NsxVpcNetworkProvider returns an instance of nsx-vpc type network provider. +func NsxVpcNetworkProvider(client client.Client) services.NetworkProvider { + return &nsxvpcNetworkProvider{ + client: client, + } +} + +func (vp *nsxvpcNetworkProvider) HasLoadBalancer() bool { + return true +} + +// nsxvpcNetworkProvider doesn't support VM readiness probe +// for the control plane vm network is private and probe +// would fail. +func (vp *nsxvpcNetworkProvider) SupportVMReadinessProbe() bool { + return false +} + +// GetNSXVPCSubnetSetName returns the name of the NSX VPC SubnetSet object. +func GetNSXVPCSubnetSetName(clusterName string) string { + return fmt.Sprintf("%s-subnetset", clusterName) +} + +func (vp *nsxvpcNetworkProvider) verifyNSXVPCSubnetSetStatus(ctx *vmware.ClusterContext, subnetset *nsxoperatorv1.SubnetSet) error { + clusterName := ctx.VSphereCluster.Name + namespace := ctx.VSphereCluster.Namespace + for _, condition := range subnetset.Status.Conditions { + if condition.Type == "Ready" && condition.Status != "True" { + conditions.MarkFalse(ctx.VSphereCluster, vmwarev1.ClusterNetworkReadyCondition, vmwarev1.ClusterNetworkProvisionFailedReason, clusterv1.ConditionSeverityWarning, condition.Message) + return errors.Errorf("subnetset ready status is: '%s' in cluster %s. reason: %s, message: %s", + condition.Status, types.NamespacedName{Namespace: namespace, Name: clusterName}, condition.Reason, condition.Message) + } + } + + conditions.MarkTrue(ctx.VSphereCluster, vmwarev1.ClusterNetworkReadyCondition) + return nil +} + +func (vp *nsxvpcNetworkProvider) VerifyNetworkStatus(_ context.Context, clusterCtx *vmware.ClusterContext, obj runtime.Object) error { + subnetset, ok := obj.(*nsxoperatorv1.SubnetSet) + if !ok { + return fmt.Errorf("expected NSX VPC SubnetSet but got %T", obj) + } + + return vp.verifyNSXVPCSubnetSetStatus(clusterCtx, subnetset) +} + +func (vp *nsxvpcNetworkProvider) ProvisionClusterNetwork(ctx context.Context, clusterCtx *vmware.ClusterContext) error { + log := ctrl.LoggerFrom(ctx) + + cluster := clusterCtx.VSphereCluster + name := GetNSXVPCSubnetSetName(cluster.Name) + + log.Info("Provisioning ", "subnetset", name) + defer log.Info("Finished provisioning", "subnetset", name) + + subnetset := &nsxoperatorv1.SubnetSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: cluster.Namespace, + Name: name, + }, + Spec: nsxoperatorv1.SubnetSetSpec{ + AdvancedConfig: nsxoperatorv1.AdvancedConfig{ + StaticIPAllocation: nsxoperatorv1.StaticIPAllocation{ + Enable: true, + }, + }, + }, + } + + _, err := ctrlutil.CreateOrPatch(ctx, vp.client, subnetset, func() error { + if err := ctrlutil.SetOwnerReference( + clusterCtx.VSphereCluster, + subnetset, + vp.client.Scheme(), + ); err != nil { + return errors.Wrapf( + err, + "error setting %s/%s as owner of %s/%s", + clusterCtx.VSphereCluster.Namespace, + clusterCtx.VSphereCluster.Name, + subnetset.Namespace, + subnetset.Name, + ) + } + + return nil + }) + if err != nil { + conditions.MarkFalse(clusterCtx.VSphereCluster, vmwarev1.ClusterNetworkReadyCondition, vmwarev1.ClusterNetworkProvisionFailedReason, clusterv1.ConditionSeverityWarning, err.Error()) + return errors.Wrap(err, "Failed to provision network") + } + + return vp.verifyNSXVPCSubnetSetStatus(clusterCtx, subnetset) +} + +// GetClusterNetworkName returns the name of a valid cluster network if one exists. +func (vp *nsxvpcNetworkProvider) GetClusterNetworkName(ctx context.Context, clusterCtx *vmware.ClusterContext) (string, error) { + subnetset := &nsxoperatorv1.SubnetSet{} + cluster := clusterCtx.VSphereCluster + namespacedName := types.NamespacedName{ + Namespace: cluster.Namespace, + Name: GetNSXVPCSubnetSetName(cluster.Name), + } + if err := vp.client.Get(ctx, namespacedName, subnetset); err != nil { + return "", err + } + return namespacedName.Name, nil +} + +func (vp *nsxvpcNetworkProvider) GetVMServiceAnnotations(ctx context.Context, clusterCtx *vmware.ClusterContext) (map[string]string, error) { + subnetsetName, err := vp.GetClusterNetworkName(ctx, clusterCtx) + if err != nil { + return nil, err + } + + return map[string]string{NSXVPCSubnetSetSelectorKey: subnetsetName}, nil +} + +// ConfigureVirtualMachine configures a VirtualMachine object based on the networking configuration. +func (vp *nsxvpcNetworkProvider) ConfigureVirtualMachine(_ context.Context, clusterCtx *vmware.ClusterContext, vm *vmoprv1.VirtualMachine) error { + networkName := GetNSXVPCSubnetSetName(clusterCtx.VSphereCluster.Name) + for _, vnif := range vm.Spec.NetworkInterfaces { + if vnif.NetworkType == NSXVPCTypeNetwork && vnif.NetworkName == networkName { + // expected network interface is already found + return nil + } + } + vm.Spec.NetworkInterfaces = append(vm.Spec.NetworkInterfaces, vmoprv1.VirtualMachineNetworkInterface{ + NetworkName: networkName, + NetworkType: NSXVPCTypeNetwork, + }) + return nil +} diff --git a/pkg/services/vmoperator/vmopmachine.go b/pkg/services/vmoperator/vmopmachine.go index 0f199935eb..7da32a2bb5 100644 --- a/pkg/services/vmoperator/vmopmachine.go +++ b/pkg/services/vmoperator/vmopmachine.go @@ -38,12 +38,14 @@ import ( vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" capvcontext "sigs.k8s.io/cluster-api-provider-vsphere/pkg/context" "sigs.k8s.io/cluster-api-provider-vsphere/pkg/context/vmware" + "sigs.k8s.io/cluster-api-provider-vsphere/pkg/services" infrautilv1 "sigs.k8s.io/cluster-api-provider-vsphere/pkg/util" ) // VmopMachineService reconciles VM Operator VM. type VmopMachineService struct { - Client client.Client + Client client.Client + NetworkProvider services.NetworkProvider } // GetMachinesInCluster returns a list of VSphereMachine objects belonging to the cluster. @@ -345,7 +347,15 @@ func (v *VmopMachineService) reconcileVMOperatorVM(ctx context.Context, supervis // Once the initial control plane node is ready, we can re-add the probe so // that subsequent machines do not attempt to speak to a kube-apiserver // that is not yet ready. - if infrautilv1.IsControlPlaneMachine(supervisorMachineCtx.Machine) && supervisorMachineCtx.Cluster.Status.ControlPlaneReady { + if v.NetworkProvider == nil { + return errors.Errorf( + "missing network provider for %s %s/%s", + supervisorMachineCtx.Machine.GroupVersionKind(), + supervisorMachineCtx.Machine.Namespace, + supervisorMachineCtx.Machine.Name) + } + // Network provider nsx-vpc doesn't support vm readiness probe + if v.NetworkProvider.SupportVMReadinessProbe() && infrautilv1.IsControlPlaneMachine(supervisorMachineCtx.Machine) && supervisorMachineCtx.Cluster.Status.ControlPlaneReady { vmOperatorVM.Spec.ReadinessProbe = &vmoprv1.Probe{ TCPSocket: &vmoprv1.TCPSocketAction{ Port: intstr.FromInt(defaultAPIBindPort), diff --git a/pkg/services/vmoperator/vmopmachine_test.go b/pkg/services/vmoperator/vmopmachine_test.go index bfdb046124..7de55c2286 100644 --- a/pkg/services/vmoperator/vmopmachine_test.go +++ b/pkg/services/vmoperator/vmopmachine_test.go @@ -37,6 +37,7 @@ import ( vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1" "sigs.k8s.io/cluster-api-provider-vsphere/pkg/context/fake" "sigs.k8s.io/cluster-api-provider-vsphere/pkg/context/vmware" + network "sigs.k8s.io/cluster-api-provider-vsphere/pkg/services/network" "sigs.k8s.io/cluster-api-provider-vsphere/pkg/util" ) @@ -118,7 +119,7 @@ var _ = Describe("VirtualMachine tests", func() { clusterContext, controllerManagerContext := util.CreateClusterContext(cluster, vsphereCluster) supervisorMachineContext = util.CreateMachineContext(clusterContext, machine, vsphereMachine) supervisorMachineContext.ControllerManagerContext = controllerManagerContext - vmService = VmopMachineService{Client: controllerManagerContext.Client} + vmService = VmopMachineService{Client: controllerManagerContext.Client, NetworkProvider: network.DummyLBNetworkProvider()} }) Context("Reconcile VirtualMachine", func() { @@ -614,7 +615,7 @@ var _ = Describe("GetMachinesInCluster", func() { } controllerManagerContext := fake.NewControllerManagerContext(initObjs...) - vmService := VmopMachineService{Client: controllerManagerContext.Client} + vmService := VmopMachineService{Client: controllerManagerContext.Client, NetworkProvider: network.DummyLBNetworkProvider()} It("returns a list of VMs belonging to the cluster", func() { objs, err := vmService.GetMachinesInCluster(context.TODO(), diff --git a/test/go.sum b/test/go.sum index 90dcd39527..7b104bb80b 100644 --- a/test/go.sum +++ b/test/go.sum @@ -629,6 +629,8 @@ github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXV github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/vmware-tanzu/net-operator-api v0.0.0-20231019160108-42131d6e8360 h1:yG158jviUd3wRqCTJcSDzp+prUZWtSA9dhfm/Rf8m9M= github.com/vmware-tanzu/net-operator-api v0.0.0-20231019160108-42131d6e8360/go.mod h1:dtVG693FvGuOSxJvTaKRVGU0EJR8yvLG3E2VaDDHILM= +github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0 h1:HdnQb/X9vJ8a5WQ03g/0nDr9igIIK1fF6wO5wOtkJT4= +github.com/vmware-tanzu/nsx-operator/pkg/apis v0.1.0/go.mod h1:Q4JzNkNMvjo7pXtlB5/R3oME4Nhah7fAObWgghVmtxk= github.com/vmware-tanzu/vm-operator/api v1.8.5 h1:E8rpRdV8+cNp/eNZ/QUHvlrbpPh8uk6bKqwEEmGWe64= github.com/vmware-tanzu/vm-operator/api v1.8.5/go.mod h1:SXaSFtnw2502Tzy0bfQVHrvbFDijR96r1ihUYQWPOK8= github.com/vmware-tanzu/vm-operator/external/ncp v0.0.0-20231214185006-5477585eebfd h1:qdfVf7KFW+XX7+D4xC/mlBpRA9+B+opdDPxGdqjxO+4=