Merge pull request #644 from askervin/5Ba_e2e_mixed_allocations

e2e: add tests for topology-aware mixed CPU allocations

test/e2e/policies.test-suite/topology-aware/n4c16/test07-mixed-allocations/code.var.sh

@@ -0,0 +1,113 @@
+# Place pod0c0 and pod0c1 to shared pools on separate nodes.
+CONTCOUNT=2 CPU=500m create guaranteed
+report allowed
+verify "len(mems['pod0c0']) == 1" \
+       "len(mems['pod0c1']) == 1" \
+       "disjoint_sets(mems['pod0c0'], mems['pod0c1'])" \
+       "len(cpus['pod0c0']) == 4" \
+       "len(cpus['pod0c1']) == 4" \
+       "disjoint_sets(cpus['pod0c0'], cpus['pod0c1'])"
+
+# Place pod1c0 to its own node, as there is still one 4-CPU node free.
+# The placement of pod1c1 is more interesting:
+# - node0 has only 3 CPUs (CPU #0 is reserved)
+# - node1, node2 and node3 have containers in their shared pools
+# - shared pools with pod0c* containers have more free space than node0
+#   => pod1c0 should be place to either of those
+# - because pod1c1 should get one exclusive CPU, either of pod0c0 and
+#   pod0c1 should run in a shared pool of only 3 CPUs from now on.
+CONTCOUNT=2 CPU=1500m create guaranteed
+report allowed
+verify `# every container is placed on a single node (no socket, no root)` \
+       "[len(mems[c]) for c in mems] == [1] * len(mems)" \
+       `# pod1c0 and pod1c1 are on different nodes` \
+       "disjoint_sets(mems['pod1c0'], mems['pod1c1'])" \
+       `# either of pod0c0 and pod0c1 has only 3 CPUs, the other has 4.` \
+       "len(cpus['pod0c0']) == 3 or len(cpus['pod0c1']) == 3" \
+       "len(cpus['pod0c0']) == 4 or len(cpus['pod0c1']) == 4" \
+       `# pod1c0 and pod1c1 are allowed to use all CPUs on their nodes` \
+       "len(cpus['pod1c0']) == 4" \
+       "len(cpus['pod1c1']) == 4" \
+       `# pod1c1 should have one exclusive CPU on its node` \
+       "len(cpus['pod1c1'] - cpus['pod0c0'] - cpus['pod0c1']) == 1"
+
+# Place pod2c0 to node0, as it has largest free shared pool (3 CPUs).
+# Place pod2c1 to the node that has only either pod0c0 or pod0c1,
+# while the other one of them already shares a node with pod1c1.
+CONTCOUNT=2 CPU=2400m create guaranteed
+report allowed
+verify `# every container is placed on a single node (no socket, no root)` \
+       "[len(mems[c]) for c in mems] == [1] * len(mems)" \
+       `# pod1c1 should have kept its own exclusive CPU` \
+       "len(cpus['pod1c1'] - set.union(*[cpus[c] for c in cpus if c != 'pod1c1'])) == 1" \
+       `# pod2c0 is the only container in node0, so it happens to have 3 unshared CPUs for now` \
+       "len(cpus['pod2c0']) == 3" \
+       "len(cpus['pod2c0'] - set.union(*[cpus[c] for c in cpus if c != 'pod2c0'])) == 3" \
+       `# pod2c1 shares its node and should not have exclusive CPUs` \
+       "len(cpus['pod2c1']) == 4" \
+       "len(cpus['pod2c1'] - set.union(*[cpus[c] for c in cpus if c != 'pod2c1'])) == 0" \
+       `# pod2c1 should run in the same node as either pod0c0 or pod0c1` \
+       "mems['pod2c1'] == mems['pod0c0'] or mems['pod2c1'] == mems['pod0c1']"
+
+# pod3c0 should get 2 exclusive CPUs and 400m share from a shared pool.
+# To get that, annotate the pod to:
+# - opt-out from shared CPUs (=> opt-in to exclusive CPUs)
+# - opt-in to isolated CPUs (this should not matter, test opt-out with pod4).
+# There is only one node where the container fits: the same node as pod1c0.
+ANNOTATIONS=('prefer-shared-cpus.cri-resource-manager.intel.com/pod: "false"'
+             'prefer-isolated-cpus.cri-resource-manager.intel.com/pod: "true"')
+CONTCOUNT=1 CPU=2400m create guaranteed-annotated
+report allowed
+verify `# every container is placed on a single node (no socket, no root)` \
+       "[len(mems[c]) for c in mems] == [1] * len(mems)" \
+       `# pod3c0 and pod1c0 are placed in the same node` \
+       "mems['pod3c0'] == mems['pod1c0']" \
+       `# pod1c0 has 1 an exclusive CPU` \
+       "len(cpus['pod1c0'] - set.union(*[cpus[c] for c in cpus if c != 'pod1c0'])) == 1" \
+       `# pod3c0 has 2 exclusive CPUs` \
+       "len(cpus['pod3c0'] - set.union(*[cpus[c] for c in cpus if c != 'pod3c0'])) == 2"
+
+# Replace pod3 with pod4.
+# Test release/(re)allocate mixed pod with exclusive CPUs and
+# no-effect from isolated preference.
+# - opt-out from shared CPUs (=> opt-in to exclusive CPUs)
+# - opt-out from isolated CPUs (this does not affect getting exclusive CPUs)
+kubectl delete pods pod3 --now
+ANNOTATIONS=('prefer-shared-cpus.cri-resource-manager.intel.com/pod: "false"'
+             'prefer-isolated-cpus.cri-resource-manager.intel.com/pod: "false"')
+CONTCOUNT=1 CPU=2400m create guaranteed-annotated
+report allowed
+verify `# every container is placed on a single node (no socket, no root)` \
+       "[len(mems[c]) for c in mems] == [1] * len(mems)" \
+       `# pod4c0 and pod1c0 are placed in the same node` \
+       "mems['pod4c0'] == mems['pod1c0']" \
+       `# pod1c0 has 1 exclusive CPU` \
+       "len(cpus['pod1c0'] - set.union(*[cpus[c] for c in cpus if c != 'pod1c0'])) == 1" \
+       `# pod4c0 has 2 exclusive CPUs` \
+       "len(cpus['pod4c0'] - set.union(*[cpus[c] for c in cpus if c != 'pod4c0'])) == 2"
+
+# Replace pod1 with pod5.
+# pod1 implicitly opted-in to exlusive CPUs due to 1500 mCPU request.
+# Now explicitly opt-out of it by opting-in to shared-cpus.
+kubectl delete pods pod1 --now
+# Make sure that shared pool size increased correctly after mixed pod deletion.
+verify `# pod0c0 or pod0c1 shared a node with pod1c1 and had only 3 CPUs` \
+       "len(cpus['pod0c0']) == 4" \
+       "len(cpus['pod0c1']) == 4"
+
+ANNOTATIONS=('prefer-shared-cpus.cri-resource-manager.intel.com/pod: "true"')
+CONTCOUNT=2 CPU=1500m create guaranteed-annotated
+report allowed
+verify `# every container is placed on a single node (no socket, no root)` \
+       "[len(mems[c]) for c in mems] == [1] * len(mems)" \
+       `# pod5c0 should share a node with pod0c0 or pod0c1 and have access to all CPUs` \
+       "mems['pod5c0'] == mems['pod0c0'] or mems['pod5c0'] == mems['pod0c1']" \
+       "len(cpus['pod5c0']) == 4" \
+       "len(cpus['pod0c0']) == 4" \
+       "len(cpus['pod0c1']) == 4" \
+       `# pod5c1 should run in a node with pod4c0 (this is where pod1c0 used to be)` \
+       "mems['pod5c1'] == mems['pod4c0']" \
+       "len(cpus['pod5c1']) == 2" \
+       `# pod5c0 and pod5c1 share a node with another container => all their CPUs should be shared` \
+       "len(cpus['pod5c0'] - set.union(*[cpus[c] for c in cpus if c != 'pod5c0'])) == 0" \
+       "len(cpus['pod5c1'] - set.union(*[cpus[c] for c in cpus if c != 'pod5c1'])) == 0"

test/e2e/policies.test-suite/topology-aware/n4c16/test07-mixed-allocations/guaranteed-annotated.yaml.in

@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: ${NAME}
+  $(if [ -n "${ANNOTATIONS[0]}" ]; then echo "
+  annotations:
+    $(for annotation in "${ANNOTATIONS[@]}"; do echo "
+    $annotation
+    "; done)
+  "; fi)
+  labels:
+    app: ${NAME}
+spec:
+  containers:
+  $(for contnum in $(seq 1 ${CONTCOUNT}); do echo "
+  - name: ${NAME}c$(( contnum - 1 ))
+    image: busybox
+    imagePullPolicy: IfNotPresent
+    command:
+      - sh
+      - -c
+      - echo ${NAME}c$(( contnum - 1 )) \$(sleep inf)
+    resources:
+      requests:
+        cpu: ${CPU}
+        memory: '${MEM}'
+      limits:
+        cpu: ${CPU}
+        memory: '${MEM}'
+  "; done )
+  terminationGracePeriodSeconds: 1