WIP, POC: Use v1.multus-cni.io/default-network and NSE for default ne…

…twork

Signed-off-by: Or Shoval <oshoval@redhat.com>

pkg/config/types.go

@@ -11,7 +11,12 @@ const (
 	NetworkRolePrimary NetworkRole = "primary"
 )
 
-const OVNPrimaryNetworkIPAMClaimAnnotation = "k8s.ovn.org/primary-udn-ipamclaim"
+//const OVNPrimaryNetworkIPAMClaimAnnotation = "k8s.ovn.org/primary-udn-ipamclaim"
+
+const (
+	MultusDefaultNetwork = "v1.multus-cni.io/default-network"
+	DefaultNetworkName   = "ovn-kubernetes"
+)
 
 type RelevantConfig struct {
 	Name               string      `json:"name"`

pkg/ipamclaimswebhook/podmutator.go

@@ -109,7 +109,24 @@ func (a *IPAMClaimsValet) Handle(ctx context.Context, request admission.Request)
 		if newPod == nil {
 			newPod = pod.DeepCopy()
 		}
-		updatePodWithOVNPrimaryNetworkIPAMClaimAnnotation(newPod, newPrimaryNetworkIPAMClaimName)
+		// add this only when a claim need to be added to NSE
+		updatePodWithDefaultNetworkAnnotation(newPod)
+
+		nseChanged, updatedNetworkSelectionElements, err :=
+			ensureIPAMClaimRefAtPrimaryNetworkSelectionElements(ctx, a.Client, newPrimaryNetworkIPAMClaimName, networkSelectionElements)
+		if err != nil {
+			return admission.Errored(http.StatusInternalServerError, err)
+		}
+		if nseChanged {
+			log.Info("new pod annotations for default network", "pod", updatedNetworkSelectionElements)
+
+			if newPod == nil {
+				newPod = pod.DeepCopy()
+			}
+			if err := updatePodSelectionElements(newPod, updatedNetworkSelectionElements); err != nil {
+				return admission.Errored(http.StatusInternalServerError, err)
+			}
+		}
 	}
 
 	if newPod != nil {
@@ -159,8 +176,8 @@ func updatePodSelectionElements(pod *corev1.Pod, networks []*v1.NetworkSelection
 	return nil
 }
 
-func updatePodWithOVNPrimaryNetworkIPAMClaimAnnotation(pod *corev1.Pod, primaryNetworkIPAMClaimName string) {
-	pod.Annotations[config.OVNPrimaryNetworkIPAMClaimAnnotation] = primaryNetworkIPAMClaimName
+func updatePodWithDefaultNetworkAnnotation(pod *corev1.Pod) {
+	pod.Annotations[config.MultusDefaultNetwork] = "default/" + config.DefaultNetworkName
 }
 
 func ensureIPAMClaimRefAtNetworkSelectionElements(ctx context.Context,
@@ -233,12 +250,87 @@ func ensureIPAMClaimRefAtNetworkSelectionElements(ctx context.Context,
 	return hasChangedNetworkSelectionElements, nil
 }
 
+func ensureIPAMClaimRefAtPrimaryNetworkSelectionElements(ctx context.Context,
+	cli client.Client, claimName string,
+	networkSelectionElements []*v1.NetworkSelectionElement) (bool, []*v1.NetworkSelectionElement, error) {
+	log := logf.FromContext(ctx)
+
+	// TODO hack
+	// networkName := "passtnet"
+
+	for i, networkSelectionElement := range networkSelectionElements {
+		nadName := fmt.Sprintf("%s/%s", networkSelectionElement.Namespace, networkSelectionElement.Name)
+		if nadName != "default/"+config.DefaultNetworkName {
+			continue
+		}
+
+		log.Info("found ovn-kubernetes NAD as part of NSE, checking if update is needed")
+		nadKey := types.NamespacedName{
+			Namespace: networkSelectionElement.Namespace,
+			Name:      networkSelectionElement.Name,
+		}
+
+		nad := v1.NetworkAttachmentDefinition{}
+		if err := cli.Get(context.Background(), nadKey, &nad); err != nil {
+			if k8serrors.IsNotFound(err) {
+				log.Info("NAD not found, will hang on scheduler", "NAD", nadName)
+				return false, networkSelectionElements, nil
+			}
+			return false, networkSelectionElements, err
+		}
+
+		pluginConfig, err := config.NewConfig(nad.Spec.Config)
+		if err != nil {
+			return false, networkSelectionElements, err
+		}
+
+		// TODO hack
+		// we need to verify externally that the UDN one has primary, it is not this network (this one is the default one)
+
+		// if !pluginConfig.AllowPersistentIPs {
+		// 	continue
+		// }
+
+		log.Info(
+			"will request persistent IPs for primary network (via default network)",
+			"NAD", nadName,
+			"network", pluginConfig.Name,
+		)
+
+		hasChangedNetworkSelectionElements := false
+		if networkSelectionElements[i].IPAMClaimReference != claimName {
+			networkSelectionElements[i].IPAMClaimReference = claimName
+			log.Info(
+				"requesting claim for primary network via deafult network",
+				"NAD", nadName,
+				"network", pluginConfig.Name,
+				"claim", networkSelectionElement.IPAMClaimReference,
+			)
+			hasChangedNetworkSelectionElements = true
+		}
+		return hasChangedNetworkSelectionElements, networkSelectionElements, nil
+	}
+
+	defaultNetworkNSE := &v1.NetworkSelectionElement{
+		Name:               config.DefaultNetworkName,
+		Namespace:          "default",
+		IPAMClaimReference: claimName,
+	}
+	networkSelectionElements = append(networkSelectionElements, defaultNetworkNSE)
+	log.Info("appending NSE for primary network", "defaultNetworkNSE", networkSelectionElements)
+
+	return true, networkSelectionElements, nil
+}
+
 func findNewPrimaryNetworkIPAMClaimName(ctx context.Context,
 	cli client.Client, pod *corev1.Pod, vmName string) (string, error) {
 	log := logf.FromContext(ctx)
-	if pod.Annotations[config.OVNPrimaryNetworkIPAMClaimAnnotation] != "" {
-		return "", nil
-	}
+
+	// change this condition, check it outside for the new annotation
+	// if pod.Annotations[config.OVNPrimaryNetworkIPAMClaimAnnotation] != "" {
+	// 	return "", nil
+	// }
+
 	primaryNetworkNAD, err := udn.FindPrimaryNetwork(ctx, cli, pod.Namespace)
 	if err != nil {
 		return "", err