Persist IPAM state to local file and use across restarts

Persist IPAM state to a file in /var/run (by default), and use this to recover state across a restart. Note no state needs to be preserved across a reboot, since all containers are also restarted. Removes use of docker/CRI and Kubernetes API from ipamd.
aws · May 19, 2020 · ac2f477 · ac2f477
1 parent 301c2a6
commit ac2f477
Show file tree

Hide file tree

Showing 20 changed files with 2,282 additions and 516 deletions.
diff --git a/cmd/aws-k8s-agent/main.go b/cmd/aws-k8s-agent/main.go
@@ -18,7 +18,6 @@ import (
 
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/ipamd"
-	"github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger"
 )
 
@@ -39,21 +38,12 @@ func _main() int {
 
 	log.Infof("Starting L-IPAMD %s  ...", version)
 
-	kubeClient, err := k8sapi.CreateKubeClient()
-	if err != nil {
-		log.Errorf("Failed to create client: %v", err)
-		return 1
-	}
-
-	discoverController := k8sapi.NewController(kubeClient)
-	go discoverController.DiscoverLocalK8SPods()
-
 	eniConfigController := eniconfig.NewENIConfigController()
 	if ipamd.UseCustomNetworkCfg() {
 		go eniConfigController.Start()
 	}
 
-	ipamContext, err := ipamd.New(discoverController, eniConfigController)
+	ipamContext, err := ipamd.New(eniConfigController)
 
 	if err != nil {
 		log.Errorf("Initialization failure: %v", err)

diff --git a/config/next/aws-k8s-cni-cn.yaml b/config/next/aws-k8s-cni-cn.yaml
@@ -0,0 +1,163 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: aws-node
+rules:
+  - apiGroups:
+      - crd.k8s.amazonaws.com
+    resources:
+      - eniconfigs
+    verbs:
+      - get
+      - list
+      - watch
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: aws-node
+  namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: aws-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aws-node
+subjects:
+  - kind: ServiceAccount
+    name: aws-node
+    namespace: kube-system
+
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: aws-node
+  namespace: kube-system
+  labels:
+    k8s-app: aws-node
+spec:
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: "10%"
+  selector:
+    matchLabels:
+      k8s-app: aws-node
+  template:
+    metadata:
+      labels:
+        k8s-app: aws-node
+    spec:
+      priorityClassName: system-node-critical
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: "beta.kubernetes.io/os"
+                    operator: In
+                    values:
+                      - linux
+                  - key: "beta.kubernetes.io/arch"
+                    operator: In
+                    values:
+                      - amd64
+                  - key: "eks.amazonaws.com/compute-type"
+                    operator: NotIn
+                    values:
+                      - fargate
+              - matchExpressions:
+                  - key: "kubernetes.io/os"
+                    operator: In
+                    values:
+                      - linux
+                  - key: "kubernetes.io/arch"
+                    operator: In
+                    values:
+                      - amd64
+                  - key: "eks.amazonaws.com/compute-type"
+                    operator: NotIn
+                    values:
+                      - fargate
+      serviceAccountName: aws-node
+      hostNetwork: true
+      tolerations:
+        - operator: Exists
+      containers:
+        - image: 961992271922.dkr.ecr.cn-northwest-1.amazonaws.com.cn/amazon-k8s-cni:v1.6.1
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 61678
+              name: metrics
+          name: aws-node
+          readinessProbe:
+            exec:
+              command: ["/app/grpc-health-probe", "-addr=:50051"]
+            initialDelaySeconds: 35
+          livenessProbe:
+            exec:
+              command: ["/app/grpc-health-probe", "-addr=:50051"]
+            initialDelaySeconds: 35
+          env:
+            - name: AWS_VPC_K8S_CNI_LOGLEVEL
+              value: DEBUG
+            - name: AWS_VPC_K8S_CNI_VETHPREFIX
+              value: eni
+            - name: AWS_VPC_ENI_MTU
+              value: "9001"
+            - name: MY_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          resources:
+            requests:
+              cpu: 10m
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+            - mountPath: /host/var/log
+              name: log-dir
+            - mountPath: /var/run/aws-routed-eni
+              name: datadir
+      volumes:
+        - name: cni-bin-dir
+          hostPath:
+            path: /opt/cni/bin
+        - name: cni-net-dir
+          hostPath:
+            path: /etc/cni/net.d
+        - name: log-dir
+          hostPath:
+            path: /var/log
+        - name: datadir
+          hostPath:
+            path: /var/run/aws-routed-eni
+            type: DirectoryOrCreate
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: eniconfigs.crd.k8s.amazonaws.com
+spec:
+  scope: Cluster
+  group: crd.k8s.amazonaws.com
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+  names:
+    plural: eniconfigs
+    singular: eniconfig
+    kind: ENIConfig
diff --git a/config/next/aws-k8s-cni-us-gov-east-1.yaml b/config/next/aws-k8s-cni-us-gov-east-1.yaml
@@ -0,0 +1,163 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: aws-node
+rules:
+  - apiGroups:
+      - crd.k8s.amazonaws.com
+    resources:
+      - eniconfigs
+    verbs:
+      - get
+      - list
+      - watch
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: aws-node
+  namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: aws-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aws-node
+subjects:
+  - kind: ServiceAccount
+    name: aws-node
+    namespace: kube-system
+
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: aws-node
+  namespace: kube-system
+  labels:
+    k8s-app: aws-node
+spec:
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: "10%"
+  selector:
+    matchLabels:
+      k8s-app: aws-node
+  template:
+    metadata:
+      labels:
+        k8s-app: aws-node
+    spec:
+      priorityClassName: system-node-critical
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: "beta.kubernetes.io/os"
+                    operator: In
+                    values:
+                      - linux
+                  - key: "beta.kubernetes.io/arch"
+                    operator: In
+                    values:
+                      - amd64
+                  - key: "eks.amazonaws.com/compute-type"
+                    operator: NotIn
+                    values:
+                      - fargate
+              - matchExpressions:
+                  - key: "kubernetes.io/os"
+                    operator: In
+                    values:
+                      - linux
+                  - key: "kubernetes.io/arch"
+                    operator: In
+                    values:
+                      - amd64
+                  - key: "eks.amazonaws.com/compute-type"
+                    operator: NotIn
+                    values:
+                      - fargate
+      serviceAccountName: aws-node
+      hostNetwork: true
+      tolerations:
+        - operator: Exists
+      containers:
+        - image: 151742754352.dkr.ecr.us-gov-east-1.amazonaws.com/amazon-k8s-cni:v1.6.1
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 61678
+              name: metrics
+          name: aws-node
+          readinessProbe:
+            exec:
+              command: ["/app/grpc-health-probe", "-addr=:50051"]
+            initialDelaySeconds: 35
+          livenessProbe:
+            exec:
+              command: ["/app/grpc-health-probe", "-addr=:50051"]
+            initialDelaySeconds: 35
+          env:
+            - name: AWS_VPC_K8S_CNI_LOGLEVEL
+              value: DEBUG
+            - name: AWS_VPC_K8S_CNI_VETHPREFIX
+              value: eni
+            - name: AWS_VPC_ENI_MTU
+              value: "9001"
+            - name: MY_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          resources:
+            requests:
+              cpu: 10m
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+            - mountPath: /host/var/log
+              name: log-dir
+            - mountPath: /var/run/aws-routed-eni
+              name: datadir
+      volumes:
+        - name: cni-bin-dir
+          hostPath:
+            path: /opt/cni/bin
+        - name: cni-net-dir
+          hostPath:
+            path: /etc/cni/net.d
+        - name: log-dir
+          hostPath:
+            path: /var/log
+        - name: datadir
+          hostPath:
+            path: /var/run/aws-routed-eni
+            type: DirectoryOrCreate
+
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: eniconfigs.crd.k8s.amazonaws.com
+spec:
+  scope: Cluster
+  group: crd.k8s.amazonaws.com
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+  names:
+    plural: eniconfigs
+    singular: eniconfig
+    kind: ENIConfig