diff --git a/.circleci/config.yml b/.circleci/config.yml index a727e2b18..99a053a86 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -66,6 +66,7 @@ jobs: NONINTERACTIVE=1 \ NO_VM_CONSOLE=1 \ INJECT_LOCAL_IMAGE=1 \ + VIRTLET_DEMO_RELEASE=master \ BASE_LOCATION="$PWD" \ deploy/demo.sh @@ -73,4 +74,4 @@ jobs: name: Run e2e tests command: | build/portforward.sh 8080& - tests/e2e/e2e.sh + _output/virtlet-e2e-tests -test.v diff --git a/Dockerfile b/Dockerfile index 7677567e9..fb8b6de55 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && \ apt-get install -y libvirt-bin libguestfs0 libguestfs-tools genisoimage \ openssl qemu-kvm qemu-system-x86 python-libvirt \ - netbase iproute2 iptables ebtables && \ + netbase iproute2 iptables ebtables vncsnapshot && \ apt-get clean RUN mkdir -p /var/lib/virtlet/volumes /opt/cni/bin && \ @@ -26,3 +26,4 @@ COPY _output/virtlet /usr/local/bin COPY _output/vmwrapper / COPY _output/criproxy / COPY _output/virtlet_log / +COPY _output/virtlet-e2e-tests / diff --git a/Makefile.am b/Makefile.am index 2a8424430..31fe16823 100644 --- a/Makefile.am +++ b/Makefile.am @@ -33,11 +33,12 @@ all: patch -d vendor/github.com/libvirt/libvirt-go-xml -p1 -b -i $(CURDIR)/libvirt-xml-go.patch; \ fi mkdir -p $(builddir)/_output $(builddir)/contrib/images/libvirt/_output - go build -o $(builddir)/_output/virtlet ./cmd/virtlet - go build -o $(builddir)/_output/vmwrapper ./cmd/vmwrapper - go build -o $(builddir)/_output/criproxy ./cmd/criproxy - go build -o $(builddir)/_output/flexvolume_driver ./cmd/flexvolume_driver - go build -o $(builddir)/_output/virtlet_log ./cmd/virtlet_log + go build -i -o $(builddir)/_output/virtlet ./cmd/virtlet + go build -i -o $(builddir)/_output/vmwrapper ./cmd/vmwrapper + go build -i -o $(builddir)/_output/criproxy ./cmd/criproxy + go build -i -o $(builddir)/_output/flexvolume_driver ./cmd/flexvolume_driver + go build -i -o $(builddir)/_output/virtlet_log ./cmd/virtlet_log + go test -i -c -o $(builddir)/_output/virtlet-e2e-tests ./tests/e2e verify-glide-installation: @which glide || go get github.com/Masterminds/glide diff --git a/glide.lock b/glide.lock index 59b308625..51b7165e6 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 4dbb56bd2f62c42b2e4b8a0030bf68c43814678defb0afeed48a08e1f2660e7e -updated: 2017-08-04T09:14:47.288882682+02:00 +hash: e04d6be0b6326f064beccc605e1b1ca51cfa784ff2fd66c3d96fc1384de3d510 +updated: 2017-08-20T03:59:07.425226591-07:00 imports: - name: cloud.google.com/go version: 3b1ae45394a234c385be014e9a488f2bb6eef821 @@ -300,6 +300,8 @@ imports: version: a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4 subpackages: - simplelru +- name: github.com/howeyc/gopass + version: bf9dde6d0d2c004a008c27aaee91170c786f6db8 - name: github.com/hpcloud/tail version: faf842bde7ed83bbc3c65a2c454fae39bc29a95f subpackages: @@ -307,6 +309,8 @@ imports: - util - watch - winfile +- name: github.com/imdario/mergo + version: 6633656539c1639d9d78127b7d47c622b5d7b6dc - name: github.com/jmespath/go-jmespath version: 3433f3ea46d9f8019119e7dd41274e112a2359a9 - name: github.com/jonboulle/clockwork @@ -337,6 +341,37 @@ imports: version: ad45545899c7b13c020ea92b2072220eefad42b8 - name: github.com/nu7hatch/gouuid version: 179d4d0c4d8d407a32af483c2354df1d2c91e6c3 +- name: github.com/onsi/ginkgo + version: 7f8ab55aaf3b86885aa55b762e803744d1674700 + subpackages: + - config + - internal/codelocation + - internal/containernode + - internal/failer + - internal/leafnodes + - internal/remote + - internal/spec + - internal/specrunner + - internal/suite + - internal/testingtproxy + - internal/writer + - reporters + - reporters/stenographer + - types +- name: github.com/onsi/gomega + version: 2152b45fa28a361beba9aab0885972323a444e28 + subpackages: + - format + - internal/assertion + - internal/asyncassertion + - internal/oraclematcher + - internal/testingtsupport + - matchers + - matchers/support/goraph/bipartitegraph + - matchers/support/goraph/edge + - matchers/support/goraph/node + - matchers/support/goraph/util + - types - name: github.com/opencontainers/runc version: d223e2adae83f62d58448a799a5da05730228089 subpackages: @@ -415,6 +450,14 @@ imports: version: 03efcb870d84809319ea509714dd6d19a1498483 subpackages: - errorutil +- name: golang.org/x/crypto + version: d172538b2cfce0c13cee31e647d0367aa8cd2486 + subpackages: + - curve25519 + - ed25519 + - ed25519/internal/edwards25519 + - ssh + - ssh/terminal - name: golang.org/x/exp version: 292a51b8d262487dab23a588950e8052d63d9113 subpackages: @@ -648,8 +691,12 @@ imports: - rest - rest/watch - testing + - tools/auth - tools/cache + - tools/clientcmd - tools/clientcmd/api + - tools/clientcmd/api/latest + - tools/clientcmd/api/v1 - tools/metrics - tools/record - tools/remotecommand @@ -657,6 +704,7 @@ imports: - util/cert - util/exec - util/flowcontrol + - util/homedir - util/integer - util/workqueue - name: k8s.io/kubernetes diff --git a/glide.yaml b/glide.yaml index 17769b70c..1b39b9fab 100644 --- a/glide.yaml +++ b/glide.yaml @@ -33,3 +33,5 @@ import: # version: 56fd84210219dbdfe6501fb2085adfb8e61f2bc1 - package: github.com/jonboulle/clockwork version: bcac9884e7502bb2b474c0339d889cb981a2f27f +- package: github.com/onsi/ginkgo +- package: github.com/onsi/gomega diff --git a/test.sh b/test.sh index 47f6d8201..796d3ac26 100755 --- a/test.sh +++ b/test.sh @@ -22,4 +22,4 @@ fi docker build -t mirantis/virtlet . VIRTLET_DEMO_RELEASE=master NONINTERACTIVE=1 NO_VM_CONSOLE=1 INJECT_LOCAL_IMAGE=1 BASE_LOCATION="${SCRIPT_DIR}" deploy/demo.sh -tests/e2e/e2e.sh +./_output/virtlet-e2e-tests -test.v diff --git a/tests/e2e/basic_test.go b/tests/e2e/basic_test.go new file mode 100644 index 000000000..9e5e14176 --- /dev/null +++ b/tests/e2e/basic_test.go @@ -0,0 +1,172 @@ +/* +Copyright 2017 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "encoding/json" + "fmt" + "path" + "strings" + "time" + + . "github.com/onsi/gomega" + + "github.com/Mirantis/virtlet/tests/e2e/framework" + . "github.com/Mirantis/virtlet/tests/e2e/ginkgo-ext" +) + +var _ = Describe("Basic cirros tests", func() { + var ( + vm *framework.VMInterface + vmPod *framework.PodInterface + ) + + BeforeAll(func() { + vm = controller.VM("cirros-vm") + vm.Create(framework.VMOptions{ + Image: *cirrosLocation, + SSHKey: sshPublicKey, + VCPUCount: 1, + DiskDriver: "virtio", + Limits: map[string]string{ + "memory": "128Mi", + }, + }, time.Minute*5, nil) + var err error + vmPod, err = vm.Pod() + Expect(err).NotTo(HaveOccurred()) + }) + + AfterAll(func() { + deleteVM(vm) + }) + + Context("VM guest OS", func() { + var ssh framework.Executor + scheduleWaitSSH(&vm, &ssh) + + It("Should have default route", func() { + Expect(framework.ExecSimple(ssh, "ip r")).To(SatisfyAll( + ContainSubstring("default via"), + ContainSubstring("src "+vmPod.Pod.Status.PodIP), + )) + }) + + It("Should have internet connectivity", func(done Done) { + defer close(done) + Expect(framework.ExecSimple(ssh, "ping -c1 8.8.8.8")).To(ContainSubstring( + "1 packets transmitted, 1 packets received, 0% packet loss")) + }, 5) + + Context("With nginx server", func() { + var nginxPod *framework.PodInterface + + BeforeAll(func() { + p, err := controller.RunPod("nginx", "nginx", nil, time.Minute*4, 80) + Expect(err).NotTo(HaveOccurred()) + Expect(p).NotTo(BeNil()) + nginxPod = p + }) + + AfterAll(func() { + Expect(nginxPod.Delete()).To(Succeed()) + }) + + It("Should be able to access another k8s endpoint", func(done Done) { + defer close(done) + cmd := fmt.Sprintf("curl -s --connect-timeout 5 http://nginx.%s.svc.cluster.local", controller.Namespace()) + Eventually(func() (string, error) { + return framework.ExecSimple(ssh, cmd) + }, 60).Should(ContainSubstring("Thank you for using nginx.")) + }, 60*5) + }) + + It("Should have hostname equal to the pod name", func() { + Expect(framework.ExecSimple(ssh, "hostname")).To(Equal(vmPod.Pod.Name)) + }) + + It("Should have CPU count that was specified for the pod", func() { + checkCPUCount(vm, ssh, 1) + }) + }) + + Context("Virtlet logs", func() { + var ( + filename string + sandboxID string + nodeExecutor framework.Executor + ) + + BeforeAll(func() { + virtletPod, err := vm.VirtletPod() + Expect(err).NotTo(HaveOccurred()) + nodeExecutor, err = virtletPod.DinDNodeExecutor() + Expect(err).NotTo(HaveOccurred()) + + domain, err := vm.Domain() + Expect(err).NotTo(HaveOccurred()) + var logPath string + for _, serial := range domain.Devices.Serials { + if serial.Type == "file" { + logPath = serial.Source.Path + break + } + } + Expect(logPath).NotTo(BeEmpty()) + var dir string + dir, filename = path.Split(logPath) + sandboxID = path.Base(dir) + }) + + It("Should contain login string in VM log", func() { + out := do(framework.ExecSimple(nodeExecutor, "cat", + fmt.Sprintf("/var/log/virtlet/vms/%s/%s", sandboxID, filename))).(string) + Expect(strings.Count(out, + "login as 'cirros' user. default password: 'cubswin:)'. use 'sudo' for root.", + )).To(Equal(1)) + }) + + It("Should contain login string in pod log and each line of that log must be a valid JSON", func() { + out := do(framework.ExecSimple(nodeExecutor, "cat", + fmt.Sprintf("/var/log/pods/%s/%s", sandboxID, filename))).(string) + found := 0 + for _, line := range strings.Split(out, "\n") { + var entry map[string]string + Expect(json.Unmarshal([]byte(line), &entry)).To(Succeed()) + if strings.HasPrefix(entry["log"], "login as 'cirros' user. default password") { + found++ + } + } + Expect(found).To(Equal(1)) + }) + }) + + It("Should provide VNC interface", func(done Done) { + defer close(done) + pod, err := vm.VirtletPod() + Expect(err).NotTo(HaveOccurred()) + + virtletPodExecutor, err := pod.Container("virtlet") + Expect(err).NotTo(HaveOccurred()) + + display, err := vm.VirshCommand("vncdisplay", "") + Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("Taking VNC display snapshot from %s", display)) + do(framework.ExecSimple(virtletPodExecutor, "vncsnapshot", "-allowblank", display, "/vm.jpg")) + }, 60) +}) diff --git a/tests/e2e/ceph_test.go b/tests/e2e/ceph_test.go new file mode 100644 index 000000000..eeab27a1e --- /dev/null +++ b/tests/e2e/ceph_test.go @@ -0,0 +1,258 @@ +/* +Copyright 2017 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "regexp" + "time" + + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/pkg/api/v1" + + "github.com/Mirantis/virtlet/tests/e2e/framework" + . "github.com/Mirantis/virtlet/tests/e2e/ginkgo-ext" +) + +const cephContainerName = "ceph_cluster" + +var _ = Describe("Ceph volumes tests", func() { + var ( + monitorIP string + secret string + ) + + BeforeAll(func() { + monitorIP, secret = setupCeph() + }) + + AfterAll(func() { + container, err := controller.DockerContainer(cephContainerName) + Expect(err).NotTo(HaveOccurred()) + container.Delete() + }) + + Context("RBD volumes", func() { + var ( + vm *framework.VMInterface + vmPod *framework.PodInterface + ) + + BeforeAll(func() { + vm = controller.VM("cirros-vm-rbd") + podCustomization := func(pod *framework.PodInterface) { + pod.Pod.Spec.Volumes = append(pod.Pod.Spec.Volumes, v1.Volume{ + Name: "test1", + VolumeSource: v1.VolumeSource{FlexVolume: cephVolume("rbd-test-image1", monitorIP, secret)}, + }) + pod.Pod.Spec.Volumes = append(pod.Pod.Spec.Volumes, v1.Volume{ + Name: "test2", + VolumeSource: v1.VolumeSource{FlexVolume: cephVolume("rbd-test-image2", monitorIP, secret)}, + }) + } + + vm.Create(framework.VMOptions{ + Image: *cirrosLocation, + SSHKey: sshPublicKey, + DiskDriver: "virtio", + Limits: map[string]string{ + "memory": "128Mi", + }, + }, time.Minute*5, podCustomization) + var err error + vmPod, err = vm.Pod() + Expect(err).NotTo(HaveOccurred()) + }) + + AfterAll(func() { + deleteVM(vm) + }) + + It("Must be attached to libvirt domain", func() { + out, err := vm.VirshCommand("domblklist", "") + Expect(err).NotTo(HaveOccurred()) + match := regexp.MustCompile("(?m:rbd-test-image[12]$)").FindAllString(out, -1) + Expect(match).To(HaveLen(2)) + }) + + Context("Mounted volumes", func() { + var ssh framework.Executor + scheduleWaitSSH(&vm, &ssh) + + It("Must be accessible from within OS", func() { + checkFilesystemAccess(ssh) + }) + }) + }) + + Context("RBD volumes defined with PV/PVC", func() { + var ( + vm *framework.VMInterface + vmPod *framework.PodInterface + ) + + BeforeAll(func() { + pv := &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rbd-pv-virtlet", + }, + Spec: v1.PersistentVolumeSpec{ + Capacity: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("10M"), + }, + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + PersistentVolumeSource: v1.PersistentVolumeSource{ + FlexVolume: cephVolume("rbd-test-image-pv", monitorIP, secret), + }, + }, + } + do(controller.PersistentVolumesClient().Create(pv)) + + pvc := &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rbd-claim", + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("10M"), + }, + }, + }, + } + do(controller.PersistentVolumeClaimsClient().Create(pvc)) + + vm = controller.VM("cirros-vm-rbd-pv") + podCustomization := func(pod *framework.PodInterface) { + pod.Pod.Spec.Volumes = append(pod.Pod.Spec.Volumes, v1.Volume{ + Name: "test", + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: "rbd-claim", + }, + }, + }) + } + + vm.Create(framework.VMOptions{ + Image: *cirrosLocation, + SSHKey: sshPublicKey, + DiskDriver: "virtio", + Limits: map[string]string{ + "memory": "128Mi", + }, + }, time.Minute*5, podCustomization) + vmPod = do(vm.Pod()).(*framework.PodInterface) + }) + + AfterAll(func() { + controller.PersistentVolumeClaimsClient().Delete("rbd-claim", nil) + controller.PersistentVolumesClient().Delete("rbd-pv-virtlet", nil) + deleteVM(vm) + }) + + It("Must be attached to libvirt domain", func() { + out := do(vm.VirshCommand("domblklist", "")).(string) + Expect(regexp.MustCompile("(?m:rbd-test-image-pv$)").MatchString(out)).To(BeTrue()) + }) + + Context("Mounted volumes", func() { + var ssh framework.Executor + scheduleWaitSSH(&vm, &ssh) + + It("Must be accessible from within OS", func() { + checkFilesystemAccess(ssh) + }) + }) + }) +}) + +func checkFilesystemAccess(ssh framework.Executor) { + do(framework.ExecSimple(ssh, "sudo /usr/sbin/mkfs.ext2 /dev/vdb")) + do(framework.ExecSimple(ssh, "sudo mount /dev/vdb /mnt")) + out := do(framework.ExecSimple(ssh, "ls -l /mnt")).(string) + Expect(out).To(ContainSubstring("lost+found")) +} + +func setupCeph() (string, string) { + nodeExecutor, err := controller.DinDNodeExecutor("kube-master") + Expect(err).NotTo(HaveOccurred()) + + route, err := framework.ExecSimple(nodeExecutor, "route") + Expect(err).NotTo(HaveOccurred()) + + match := regexp.MustCompile(`default\s+([\d.]+)`).FindStringSubmatch(route) + Expect(match).To(HaveLen(2)) + + monIP := match[1] + cephPublicNetwork := monIP + "/16" + + container, err := controller.DockerContainer(cephContainerName) + Expect(err).NotTo(HaveOccurred()) + + container.Delete() + Expect(container.PullImage("ceph/demo")).To(Succeed()) + Expect(container.Run("ceph/demo", + map[string]string{"MON_IP": monIP, "CEPH_PUBLIC_NETWORK": cephPublicNetwork}, + "host", nil, false)).To(Succeed()) + + cephContainerExecutor := container.Executor(false, "") + By("Waiting for ceph cluster") + Eventually(func() error { + _, err := framework.ExecSimple(cephContainerExecutor, "ceph", "-s") + return err + }).Should(Succeed()) + By("Ceph cluster started") + + var out string + commands := []string{ + // Adjust ceph configs + `echo -e "rbd default features = 1\nrbd default format = 2" >> /etc/ceph/ceph.conf`, + + // Add rbd pool and volume + `ceph osd pool create libvirt-pool 8 8`, + `apt-get update && apt-get install -y qemu-utils 2> /dev/null`, + `qemu-img create -f rbd rbd:libvirt-pool/rbd-test-image1 10M`, + `qemu-img create -f rbd rbd:libvirt-pool/rbd-test-image2 10M`, + `qemu-img create -f rbd rbd:libvirt-pool/rbd-test-image-pv 10M`, + + // Add user for virtlet + `ceph auth get-or-create client.libvirt`, + `ceph auth caps client.libvirt mon "allow *" osd "allow *" msd "allow *"`, + `ceph auth get-key client.libvirt`, + } + for _, cmd := range commands { + out = do(framework.ExecSimple(cephContainerExecutor, "/bin/bash", "-c", cmd)).(string) + } + return monIP, out +} + +func cephVolume(volume, monitorIP, secret string) *v1.FlexVolumeSource { + return &v1.FlexVolumeSource{ + Driver: "virtlet/flexvolume_driver", + Options: map[string]string{ + "type": "ceph", + "monitor": monitorIP + ":6789", + "user": "libvirt", + "secret": secret, + "volume": volume, + "pool": "libvirt-pool", + }, + } +} diff --git a/tests/e2e/common.go b/tests/e2e/common.go new file mode 100644 index 000000000..0be252ff2 --- /dev/null +++ b/tests/e2e/common.go @@ -0,0 +1,101 @@ +/* +Copyright 2017 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "time" + + . "github.com/onsi/gomega" + + "github.com/Mirantis/virtlet/tests/e2e/framework" + . "github.com/Mirantis/virtlet/tests/e2e/ginkgo-ext" +) + +// scheduleWaitSSH schedules SSH interface initialization before the test context starts +func scheduleWaitSSH(vm **framework.VMInterface, ssh *framework.Executor) { + BeforeAll(func() { + Eventually( + func() error { + var err error + *ssh, err = (*vm).SSH("cirros", sshPrivateKey) + if err != nil { + return err + } + _, err = framework.ExecSimple(*ssh) + return err + }, 60*5, 3).Should(Succeed()) + }) + + AfterAll(func() { + (*ssh).Close() + }) +} + +func checkCPUCount(vm *framework.VMInterface, ssh framework.Executor, cpus int) { + proc := do(framework.ExecSimple(ssh, "cat", "/proc/cpuinfo")).(string) + Expect(regexp.MustCompile(`(?m)^processor`).FindAllString(proc, -1)).To(HaveLen(cpus)) + cpuStats := do(vm.VirshCommand("domstats", "", "--vcpu")).(string) + match := regexp.MustCompile(`vcpu\.maximum=(\d+)`).FindStringSubmatch(cpuStats) + Expect(match).To(HaveLen(2)) + Expect(strconv.Atoi(match[1])).To(Equal(cpus)) +} + +func deleteVM(vm *framework.VMInterface) { + virtletPod, err := vm.VirtletPod() + Expect(err).NotTo(HaveOccurred()) + + domainName, err := vm.DomainName() + Expect(err).NotTo(HaveOccurred()) + domainName = domainName[8:21] // extract 5d3f8619-fda4 from virtlet-5d3f8619-fda4-cirros-vm + + Expect(vm.Delete(time.Minute * 2)).To(Succeed()) + + commands := map[string][]string{ + "domain": {"list", "--name"}, + "volume": {"vol-list", "--pool", "volumes"}, + "secret": {"secret-list"}, + } + + for key, cmd := range commands { + Eventually(func() error { + out, err := framework.RunVirsh(virtletPod, cmd...) + if err != nil { + return err + } + if strings.Contains(out, domainName) { + return fmt.Errorf("%s ~%s~ was not deleted", key, domainName) + } + return nil + }, "2m").Should(Succeed()) + } +} + +// do asserts that function with multiple return values doesn't fail +// considering we have func `foo(something) (something, error)` +// +// `x := do(foo(something))` is equivalent to +// val, err := fn(something) +// Expect(err).To(Succeed()) +// x = val +func do(value interface{}, extra ...interface{}) interface{} { + ExpectWithOffset(1, value, extra...).To(BeAnything()) + return value +} diff --git a/tests/e2e/constants.go b/tests/e2e/constants.go new file mode 100644 index 000000000..d8bc144b3 --- /dev/null +++ b/tests/e2e/constants.go @@ -0,0 +1,55 @@ +/* +Copyright 2017 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +const ( + defaultCirrosLocation = "download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img" + + sshPublicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+deP" + + "xdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwW" + + "zAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0" + + "334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost" + + sshPrivateKey = ` + -----BEGIN RSA PRIVATE KEY----- + MIIEpAIBAAKCAQEAmiRHBQ1xCtmW19GS0tRCGBWUWwwHEX7lY6bLUnNA3vPrFdWo + rvnXj8XZLg0cKhbQspOnRGZLLIzpAV7a4tNDBAi5Iq+hXB95CjibWyer8t7miXPc + NREaPsMpekpAIf/iUyZKe0EV4/KMpVuRNtC7G5wP3SMFswBedTz8BewOxbTYPQ58 + D6jJMRaNU4f27jjCmz+WQNZpjTbZs2zKTHyQE71cor93HLEiSjoskK+lBvo+7HP+ + 4zFc+V5N+FXdJfBsHAdN9+DHIR3JJqgwdeNofzQlk6/KC2pAzb5HwtWJ3gHwDuNY + Q5FG8T7VQRuTP6Oz+H8aKohxF/mS30cQ7RwedwIDAQABAoIBABAZa+WGMuFcOpoO + BJTKoKCdWGJuDirwowrWd/QDn6nptgsQxs6Hv9D/bCCYM/HdcizEqTrGqGFd0lRX + UOtR/3TjaFrMF0Fk9CJyKR/LM/Vo/JEsrbpJMAGQJrvkF3C1pjDjFfJrqNqnEbOP + rcoY4QIQOcPyDX1Vs4fxN61yq1RQ1qnyZ6mJkCzVi2zcrlLBOorAAUqJ0sic/I+Z + kaPuRUaX7x63McrX2N09kr+hcwsIxh9ZQf3nZp5CHJy4E6iP0hab5UtvcFJAXFzr + yBT5oi/hWCm+lfiZ2I7hyAQvVltr2uMMSUo6NbEBZbq955BO+VeUnNncltmkphDC + pePuRTkCgYEAx7O60vTqXuJpN79bYOYa+M1va1NA/dqdB427wIiP99ZtGeHrpEvy + GO2PplbgN31a/E924myWQ3Z7GvFtfGjYYqzcXHZq5c328o2oScLYc/tYdjppwMQL + jQ6B4s1uyf05PLuCuGCPfGX8cAlfPO+xBPd2RtqGEv9zF5dMEzWBNK0CgYEAxZiC + Ka+ypSjNQ4l5UrI58GaunmXIFcZIGKku/aqlOoVzDZWLXI582aycKc3K/VIgxN8P + PLurDLu+s8M55RJiZnfh59ggDqonrOyLeenE3L9EI+t6VNs3nRfSodiqT202Hr2p + F571YKhj6lFLFHGWZQHIvm0DzK14F/hUgFUNIDMCgYEAnaLN0j/p0UQ/cfXnF7IL + kGH5lWp+XuP2GERU9EHYAvaL4GZpL6OTUwIS5malTqfw7kF7wneclVwtCLOSjSXl + yN5Sg9olv4i5afVP5gmb+tFonsq1N6iIxaux82neDiuIxtvs78Wo/bUzcuyy9NLv + lNAR2RQdyVlDbFfNgUw21XECgYEAnpfmuQCtGQSjo3ZeqzIjcMFpm/bDXj60NR7t + eWoSjeL4UknZ/iLbMHbrLF5hc2sMpBcIis1x35l82ZlzCVn1IptL9SKxsDN//roo + xGQNvsPBNDdXC26bt3mcdIyLPY7BZnEBm9TYy4i8ESDIaxM0C8Qf1D95UjlU76BA + anRZQaMCgYASTlP6c0axoWe+YLP2zEk6CqYRfGuYeTBjZdVzOAeHo6j+18PzJ9/Z + 87JtpIvSfvR+u1s07Bshcbq2ppEcDRcWUDw6yCVSt60pHB+YLu/kTTeuyC3/r8BJ + A2jURl0Eel11p/WRSwxCFENaAPx+mBKvZxcXORc5ocu63S6wuhZ8KA== + -----END RSA PRIVATE KEY-----` +) diff --git a/tests/e2e/e2e.sh b/tests/e2e/e2e.sh deleted file mode 100755 index 3ba1fc58a..000000000 --- a/tests/e2e/e2e.sh +++ /dev/null @@ -1,285 +0,0 @@ -#!/bin/bash -# Copyright 2017 Mirantis -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail -set -o errtrace - -if [ $(uname) = Darwin ]; then - readlinkf(){ perl -MCwd -e 'print Cwd::abs_path shift' "$1";} -else - readlinkf(){ readlink -f "$1"; } -fi - -SCRIPT_DIR="$(cd $(dirname "$(readlinkf "${BASH_SOURCE}")"); pwd)" -virsh="${SCRIPT_DIR}/../../examples/virsh.sh" -vmssh="${SCRIPT_DIR}/../../examples/vmssh.sh" - -# provide path for kubectl -export PATH="${HOME}/.kubeadm-dind-cluster:${PATH}" - -function wait-for-pod { - local pod="${1}" - local n=180 - while true; do - local phase="$(kubectl get pod "${1}" -o jsonpath='{.status.phase}')" - if [[ ${phase} == Running ]]; then - break - fi - if ((--n == 0)); then - echo "Timed out waiting for pod ${pod}" >&2 - exit 1 - fi - sleep 1 - echo -n "." >&2 - done - echo >&2 -} - -function check-all-cleaned { - local podID="${1}" - # TODO: get rid of this sleep - # for some reason even after 'kubectl get pod' check returns 'NOT FOUND' - # domain remains defined being in "shut off" state for noticable time - sleep 60 - if "${virsh}" list --all | grep ${podID}; then - echo "domain ${podID} still listed after deletion" >&2 - exit 1 - fi - - if "${virsh}" vol-list --pool volumes | grep ${podID}; then - echo "volumes for domain ${podID} still listed after deletion" >&2 - exit 1 - fi - - if "${virsh}" secret-list | grep ${podID}; then - echo "secret(s) for domain ${podID} still listed after deletion" >&2 - exit 1 - fi -} - -function wait-for-ssh { - local vmname=${1} - local retries=10 - # dropbear inside cirros vm on Travis (thus non-KVM) is shaky - for ((i = 0; i < 4; i++)); do - while ! ../../examples/vmssh.sh "cirros@${vmname}" "echo Hello ${1}" | grep -q "Hello ${1}"; do - if ((--retries <= 0));then - echo "Timed out waiting for ssh to ${vmname}" - exit 1 - fi - echo "Waiting for ssh to ${vmname}" - sleep 1 - done - done -} - -function delete-pod-and-wait { - local pod="${1}" - kubectl delete pod "${pod}" - n=180 - while kubectl get pod "${pod}" >&/dev/null; do - if ((--n == 0)); then - echo "Timed out waiting for pod removal" >&2 - exit 1 - fi - sleep 1 - echo -n "." >&2 - done - echo >&2 - if "${virsh}" list --name|grep -- '-${pod}$'; then - echo "${pod} domain still listed after deletion" >&2 - exit 1 - fi -} - -function vmchat-short { - local vmname=${1} - wait-for-ssh ${vmname} - - count=$(../../examples/vmssh.sh "cirros@${vmname}" "ip a" | grep "eth0:" | wc -l) - if [[ ${count} != 1 ]]; then - echo "Executing 'ip a' failed. Expected 1 line but got ${count}" - exit 1 - fi -} - -function vmchat { - local vmname=${1} - vmchat-short ${vmname} - - count=$(../../examples/vmssh.sh "cirros@${vmname}" "ip r" | grep "default via" | wc -l) - if [[ ${count} != 1 ]]; then - echo "Executing 'ip r' failed. Expected 1 line but got ${count}" - exit 1 - fi - - count=$(../../examples/vmssh.sh "cirros@${vmname}" "ping -c1 8.8.8.8" | grep "1 packets transmitted, 1 packets received, 0% packet loss" | wc -l) - if [[ ${count} != 1 ]]; then - echo "Executing 'ping -c1 8.8.8.8' failed. Expected 1 line but got ${count}" - exit 1 - fi - - count=$(../../examples/vmssh.sh "cirros@${vmname}" "curl http://nginx.default.svc.cluster.local" | grep "Thank you for using nginx." | wc -l) - if [[ ${count} != 1 ]]; then - echo "Executing 'curl http://nginx.default.svc.cluster.local' failed. Expected 1 line but got ${count}" - exit 1 - fi -} - -wait-for-pod cirros-vm - -cd "${SCRIPT_DIR}" -vmchat cirros-vm - -# test logging - -virshid="$("${virsh}" list | grep "\-cirros-vm " | cut -f2 -d " ")" -logpath="$("${virsh}" dumpxml $virshid | xmllint --xpath 'string(//serial[@type="file"]/source/@path)' -)" -filename="$(echo $logpath | sed -E 's#.+/##')" -sandboxid="$(echo $logpath | sed 's#/var/log/vms/##' | sed -E 's#/.+##')" -nodeid="$(kubectl get pod -n kube-system -l runtime=virtlet -o jsonpath='{.items[0].spec.nodeName}')" - -count=$(docker exec -it ${nodeid} /bin/bash -c "cat /var/log/virtlet/vms/${sandboxid}/${filename}" | \ - grep "login as 'cirros' user. default password: 'cubswin:)'. use 'sudo' for root." | \ - wc -l) -if [[ ${count} != 1 ]]; then - echo "Checking raw log file failed. Expected 1 line but got ${count}" - exit 1 -fi - -# DIND containers have jq installed so we can use it -count=$(docker exec -it ${nodeid} /bin/bash -c "jq .log /var/log/pods/${sandboxid}/${filename}" | \ - grep "login as 'cirros' user. default password:.*'. use 'sudo' for root" | \ - wc -l) -if [[ ${count} != 1 ]]; then - echo "Checking formatted log file failed. Expected 1 line but got ${count}" - exit 1 -fi - -vm_hostname="$("${vmssh}" cirros@cirros-vm cat /etc/hostname)" -expected_hostname=cirros-vm -if [[ "${vm_hostname}" != "${expected_hostname}" ]]; then - echo "Unexpected vm hostname: ${vm_hostname} instead ${expected_hostname}" >&2 - exit 1 -fi - -# test ceph RBD - -virtlet_pod_name=$(kubectl get pods --namespace=kube-system | grep -v virtlet-log | grep virtlet | awk '{print $1}') - -# Run one-node ceph cluster -"${SCRIPT_DIR}/run_ceph.sh" "${SCRIPT_DIR}" - -# check attaching multiple RBD device specified in the pod definition -kubectl create -f "${SCRIPT_DIR}/cirros-vm-rbd-volume.yaml" -wait-for-pod cirros-vm-rbd -if [ "$(${virsh} domblklist @cirros-vm-rbd | grep rbd-test-image[12]$ | wc -l)" != "2" ]; then - echo "ceph: failed to find rbd-test-image in domblklist" >&2 - exit 1 -fi - -# check attaching rbd device specified using PV/PVC -kubectl create -f "${SCRIPT_DIR}/cirros-vm-rbd-pv-volume.yaml" -wait-for-pod cirros-vm-rbd-pv -if [ "$(${virsh} domblklist @cirros-vm-rbd-pv | grep rbd-test-image-pv$ | wc -l)" != "1" ]; then - echo "ceph: failed to find rbd-test-image-pv in domblklist" >&2 - exit 1 -fi - -# wait for login prompt to appear -vmchat-short cirros-vm-rbd - -"${vmssh}" cirros@cirros-vm-rbd 'sudo /usr/sbin/mkfs.ext2 /dev/vdb && sudo mount /dev/vdb /mnt && ls -l /mnt | grep lost+found' - -# check vnc consoles are available for both domains -if ! kubectl exec "${virtlet_pod_name}" -c virtlet --namespace=kube-system -- /bin/sh -c "apt-get install -y vncsnapshot"; then - echo "Failed to install vncsnapshot inside virtlet container" >&2 - exit 1 -fi - -# grab screenshots - -if ! kubectl exec "${virtlet_pod_name}" -c virtlet --namespace=kube-system -- /bin/sh -c "vncsnapshot :0 /domain_1.jpeg"; then - echo "Failed to attach and get screenshot for vnc console for domain with 1 id" >&2 - exit 1 -fi - -if ! kubectl exec "${virtlet_pod_name}" -c virtlet --namespace=kube-system -- /bin/sh -c "vncsnapshot :1 /domain_2.jpeg"; then - echo "Failed to attach and get screenshot for vnc console for domain with 2 id" >&2 - exit 1 -fi - -# check cpu count - -function verify-cpu-count { - local expected_count="${1}" - cirros_cpu_count="$("${vmssh}" cirros@cirros-vm grep '^processor' /proc/cpuinfo|wc -l)" - if [[ ${cirros_cpu_count} != ${expected_count} ]]; then - echo "bad cpu count for cirros-vm: ${cirros_cpu_count} instead of ${expected_count}" >&2 - exit 1 - fi -} - -verify-cpu-count 1 - -# test pod removal - -podID=$("${virsh}" poddomain @cirros-vm-rbd-pv | sed s/-cirros-vm-rbd-pv//) -delete-pod-and-wait cirros-vm-rbd-pv -check-all-cleaned ${podID} - -podID=$("${virsh}" poddomain @cirros-vm | sed s/-cirros-vm//) -delete-pod-and-wait cirros-vm -check-all-cleaned ${podID} - -# test changing vcpu count - -kubectl convert -f "${SCRIPT_DIR}/../../examples/cirros-vm.yaml" --local -o json | jq '.metadata.annotations.VirtletVCPUCount = "2" | .spec.containers[0].resources.limits.cpu = "500m"' | kubectl create -f - - -wait-for-pod cirros-vm - -# wait for login prompt to appear -vmchat-short cirros-vm - -verify-cpu-count 2 - -pod_domain="$("${virsh}" poddomain @cirros-vm)" - -# verify memory size as reported by Linux kernel inside VM - -# The boot message is: -# [ 0.000000] Memory: 109112k/130944k available (6576k kernel code, 452k absent, 21380k reserved, 6620k data, 928k init) - -mem_size_k="$("${vmssh}" cirros@cirros-vm dmesg|grep 'Memory:'|sed 's@.*/\|k .*@@g')" -expected_mem_size_k=130944 - -if [[ ${mem_size_k} != ${expected_mem_size_k} ]]; then - echo "Bad memory size (inside VM). Expected ${expected_mem_size_k}, but got ${mem_size_k}" >&2 - exit 1 -fi - -# Try stopping hung vm. We make VM hang by invoking 'halt -nf' from -# cloud-init userdata -kubectl convert -f "${SCRIPT_DIR}/../../examples/cirros-vm.yaml" --local -o json | - jq '.metadata.name="haltme"|.metadata.annotations.VirtletCloudInitUserDataScript="#!/bin/sh\n/sbin/halt -nf"' | - kubectl create -f - -wait-for-pod haltme -# FIXME: it would be better to halt the VM over ssh + wait for it to -# stop receiving pings probably. We can do it after rewriting e2e -# tests in Go. -sleep 15 -delete-pod-and-wait haltme diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go new file mode 100644 index 000000000..1356a3901 --- /dev/null +++ b/tests/e2e/e2e_test.go @@ -0,0 +1,50 @@ +/* +Copyright 2017 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "flag" + "fmt" + "testing" + "time" + + . "github.com/onsi/gomega" + + "github.com/Mirantis/virtlet/tests/e2e/framework" + . "github.com/Mirantis/virtlet/tests/e2e/ginkgo-ext" +) + +var controller *framework.Controller +var cirrosLocation = flag.String("cirros", defaultCirrosLocation, "cirros image URL (*without http(s)://*") + +func TestE2E(t *testing.T) { + SetDefaultEventuallyTimeout(time.Minute * 5) + RegisterFailHandler(Fail) + + BeforeAll(func() { + var err error + controller, err = framework.NewController("") + Expect(err).ToNot(HaveOccurred()) + + By(fmt.Sprintf("Using namespace %s", controller.Namespace())) + }) + AfterAll(func() { + controller.Finalize() + }) + + RunSpecs(t, "Virtlet E2E suite") +} diff --git a/tests/e2e/framework/common.go b/tests/e2e/framework/common.go new file mode 100644 index 000000000..ce4de9a59 --- /dev/null +++ b/tests/e2e/framework/common.go @@ -0,0 +1,133 @@ +/* +Copyright 2017 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "bytes" + "fmt" + "io" + "strings" + "time" +) + +// ErrTimeout is the timeout error returned from functions wrapped by WithTimeout +var ErrTimeout = fmt.Errorf("timeout") + +// Executor is the interface to execute shell commands in arbitrary places +type Executor interface { + io.Closer + Exec(command []string, stdin io.Reader, stdout, stderr io.Writer) (int, error) +} + +// Exec executes command with the given executor and returns stdout/stderr/exitCode as strings +func Exec(executor Executor, command []string, input string) (string, string, int, error) { + outBuf := &bytes.Buffer{} + errBuf := &bytes.Buffer{} + inBuf := bytes.NewBufferString(input) + exitCode, err := executor.Exec(command, inBuf, outBuf, errBuf) + if err != nil { + return "", "", 0, err + } + return outBuf.String(), errBuf.String(), exitCode, nil +} + +// ExecSimple is a simplified version of Exec that verifies exit code/stderr internally and returns stdout only +func ExecSimple(executor Executor, command ...string) (string, error) { + stdout, stderr, exitcode, err := Exec(executor, command, "") + if err != nil { + return "", err + } + if exitcode != 0 { + return "", fmt.Errorf("command exited with code %d, stderr: %s", exitcode, strings.TrimSpace(stderr)) + } + return strings.TrimSpace(stdout), nil +} + +func trimBlock(s string) string { + lines := strings.Split(s, "\n") + for i, line := range lines { + lines[i] = strings.TrimSpace(line) + } + return strings.Join(lines, "\n") +} + +func waitFor(f func() error, wait, poll time.Duration, waitFailure bool) error { + if poll <= 0 || wait <= 0 { + wait = time.Duration(time.Hour) + poll = 0 + } + timeout := time.After(wait) + err := f() + if err == nil && !waitFailure || err != nil && waitFailure { + return err + } + result := err + for { + select { + case <-time.After(poll): + err := f() + if err == nil && !waitFailure || err != nil && waitFailure { + return err + } + result = err + if poll == 0 { + return result + } + case <-timeout: + return result + } + } +} + +func waitForConsistentState(f func() error, timing ...time.Duration) error { + if len(timing) == 0 { + panic("timing is not provided") + } + var pollPeriod time.Duration + if len(timing) == 1 || timing[1] <= 0 { + pollPeriod = time.Duration(timing[0].Nanoseconds() / 10) + } else { + pollPeriod = timing[1] + } + if err := waitFor(f, timing[0], pollPeriod, false); err != nil { + return err + } + + if len(timing) >= 2 { + if err := waitFor(f, timing[2], pollPeriod, true); err != nil { + return err + } + } + return nil +} + +// WithTimeout adds timeout to synchronous function +func WithTimeout(timeout time.Duration, fn func() error) func() error { + return func() error { + res := make(chan error, 1) + go func() { + res <- fn() + }() + timer := time.After(timeout) + select { + case e := <-res: + return e + case <-timer: + return ErrTimeout + } + } +} diff --git a/tests/e2e/framework/controller.go b/tests/e2e/framework/controller.go new file mode 100644 index 000000000..805b9575e --- /dev/null +++ b/tests/e2e/framework/controller.go @@ -0,0 +1,208 @@ +/* +Copyright 2017 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "flag" + "fmt" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/pkg/api/v1" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + + // register standard k8s types + _ "k8s.io/kubernetes/pkg/api/install" +) + +var url = flag.String("cluster-url", "http://127.0.0.1:8080", "apiserver URL") + +// Controller is the entry point for various operations on k8s+virtlet entities +type Controller struct { + fixedNs bool + + client *typedv1.CoreV1Client + namespace *v1.Namespace + restConfig *restclient.Config +} + +// NewController creates instance of controller for specified k8s namespace. +// If namespace is empty string then namespace with random name is going to be created +func NewController(namespace string) (*Controller, error) { + config, err := clientcmd.BuildConfigFromFlags(*url, "") + if err != nil { + return nil, err + } + + clientset, err := typedv1.NewForConfig(config) + if err != nil { + return nil, err + } + + var ns *v1.Namespace + if namespace != "" { + ns, err = clientset.Namespaces().Get(namespace, metav1.GetOptions{}) + } else { + ns, err = createNamespace(clientset) + } + if err != nil { + return nil, err + } + + return &Controller{ + client: clientset, + namespace: ns, + restConfig: config, + fixedNs: namespace != "", + }, nil +} + +func createNamespace(client *typedv1.CoreV1Client) (*v1.Namespace, error) { + namespaceObj := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "virtlet-tests-", + }, + Status: v1.NamespaceStatus{}, + } + return client.Namespaces().Create(namespaceObj) +} + +// Finalize deletes random namespace that might has been created by NewController +func (c *Controller) Finalize() error { + if c.fixedNs { + return nil + } + return c.client.Namespaces().Delete(c.namespace.Name, nil) +} + +// PersistentVolumesClient returns interface for PVs +func (c *Controller) PersistentVolumesClient() typedv1.PersistentVolumeInterface { + return c.client.PersistentVolumes() +} + +// PersistentVolumeClaimsClient returns interface for PVCs +func (c *Controller) PersistentVolumeClaimsClient() typedv1.PersistentVolumeClaimInterface { + return c.client.PersistentVolumeClaims(c.namespace.Name) +} + +// VM returns interface for operations on virtlet VM pods +func (c *Controller) VM(name string) *VMInterface { + return newVMInterface(c, name) +} + +// Pod returns interface for operations on k8s pod in a given namespace. +// If namespace is an empty string then default controller namespace is used +func (c *Controller) Pod(name, namespace string) (*PodInterface, error) { + if namespace == "" { + namespace = c.namespace.Name + } + pod, err := c.client.Pods(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return newPodInterface(c, pod), nil +} + +// FindPod looks for a pod in a given namespace having specified labels and matching optional predicate function +func (c *Controller) FindPod(namespace string, labelMap map[string]string, + predicate func(podInterface *PodInterface) bool) (*PodInterface, error) { + + if namespace == "" { + namespace = c.namespace.Name + } + lst, err := c.client.Pods(namespace).List(metav1.ListOptions{LabelSelector: labels.SelectorFromSet(labelMap).String()}) + if err != nil { + return nil, err + } + for _, pod := range lst.Items { + pi := newPodInterface(c, &pod) + if predicate == nil || predicate(pi) { + return pi, nil + } + } + return nil, nil +} + +// DinDNodeExecutor returns executor in DinD container for one of k8s nodes +func (c *Controller) DinDNodeExecutor(name string) (Executor, error) { + dockerInterface, err := newDockerContainerInterface(name) + if err != nil { + return nil, err + } + return dockerInterface.Executor(false, ""), nil +} + +// DockerContainer returns interface for operations on a docker container with a given name +func (c *Controller) DockerContainer(name string) (*DockerContainerInterface, error) { + return newDockerContainerInterface(name) +} + +// Namespace returns default controller namespace name +func (c *Controller) Namespace() string { + return c.namespace.Name +} + +// RunPod is a helper method to create a pod in a simple configuration (similar to `kubectl run`) +func (c *Controller) RunPod(name, image string, command []string, timeout time.Duration, exposePorts ...int32) (*PodInterface, error) { + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{"id": name}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: name, + Image: image, + Command: command, + }, + }, + }, + } + podInterface := newPodInterface(c, pod) + if err := podInterface.Create(); err != nil { + return nil, err + } + if err := podInterface.Wait(timeout); err != nil { + return nil, err + } + if len(exposePorts) > 0 { + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{"id": name}, + }, + } + for _, port := range exposePorts { + svc.Spec.Ports = append(svc.Spec.Ports, v1.ServicePort{ + Name: fmt.Sprintf("port%d", port), + Port: port, + }) + } + _, err := c.client.Services(c.namespace.Name).Create(svc) + if err != nil { + return nil, err + } + podInterface.hasService = true + } + return podInterface, nil +} diff --git a/tests/e2e/framework/docker_interface.go b/tests/e2e/framework/docker_interface.go new file mode 100644 index 000000000..ebf4d11fb --- /dev/null +++ b/tests/e2e/framework/docker_interface.go @@ -0,0 +1,225 @@ +/* +Copyright 2017 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + "fmt" + "io" + "io/ioutil" + + "github.com/docker/docker/pkg/stdcopy" + "github.com/docker/engine-api/client" + "github.com/docker/engine-api/types" + "github.com/docker/engine-api/types/container" + "github.com/docker/engine-api/types/filters" + "github.com/docker/go-connections/nat" +) + +// DockerContainerInterface is the receiver object for docker container operations +type DockerContainerInterface struct { + client *client.Client + Name string + ID string +} + +// DockerContainerExecInterface is the receiver object for commands execution in docker container +type DockerContainerExecInterface struct { + dockerInterface *DockerContainerInterface + user string + privileged bool +} + +func newDockerContainerInterface(name string) (*DockerContainerInterface, error) { + cli, err := client.NewEnvClient() + if err != nil { + return nil, err + } + return &DockerContainerInterface{ + client: cli, + Name: name, + }, nil +} + +// PullImage pulls docker image from remote registry +func (d *DockerContainerInterface) PullImage(name string) error { + out, err := d.client.ImagePull(context.Background(), name, types.ImagePullOptions{}) + if err != nil { + return err + } + defer out.Close() + _, err = io.Copy(ioutil.Discard, out) + if err != nil { + return err + } + return nil +} + +// Delete deletes docker container +func (d *DockerContainerInterface) Delete() error { + id := d.Name + if d.ID != "" { + id = d.ID + } + return d.client.ContainerRemove(context.Background(), id, types.ContainerRemoveOptions{ + RemoveVolumes: true, + Force: true, + }) +} + +// Run starts new docker container (similar to `docker run`) +func (d *DockerContainerInterface) Run(image string, env map[string]string, network string, ports []string, privileged bool, cmd ...string) error { + var envLst []string + for key, value := range env { + envLst = append(envLst, fmt.Sprintf("%s=%s", key, value)) + } + + exposedPorts, portBindings, err := nat.ParsePortSpecs(ports) + if err != nil { + return err + } + config := &container.Config{ + ExposedPorts: exposedPorts, + Env: envLst, + Image: image, + Cmd: cmd, + } + + hostConfig := &container.HostConfig{ + NetworkMode: container.NetworkMode(network), + PortBindings: portBindings, + Privileged: privileged, + } + + ctx := context.Background() + resp, err := d.client.ContainerCreate(ctx, config, hostConfig, nil, d.Name) + if err != nil { + return err + } + if err := d.client.ContainerStart(ctx, resp.ID); err != nil { + return err + } + d.ID = resp.ID + return nil +} + +// Container returns info for the container associated with method receiver +func (d *DockerContainerInterface) Container() (*types.Container, error) { + args := filters.NewArgs() + var id string + if d.ID != "" { + args.Add("id", d.ID) + id = d.ID + } else if d.Name != "" { + args.Add("name", d.Name) + id = d.Name + } else { + return nil, nil + } + containers, err := d.client.ContainerList(context.Background(), types.ContainerListOptions{ + All: true, + Filter: args, + }) + if err != nil { + return nil, err + } + if len(containers) < 1 { + return nil, fmt.Errorf("Cannot find docker container %s", id) + } + return &containers[0], nil +} + +// Executor returns interface to run commands in docker container +func (d *DockerContainerInterface) Executor(privileged bool, user string) Executor { + return &DockerContainerExecInterface{ + user: user, + privileged: privileged, + dockerInterface: d, + } +} + +// Exec executes command in docker container +func (n *DockerContainerExecInterface) Exec(command []string, stdin io.Reader, stdout, stderr io.Writer) (int, error) { + ctx := context.Background() + cfg := types.ExecConfig{ + AttachStdout: stdout != nil, + AttachStderr: stderr != nil, + AttachStdin: stdin != nil, + Cmd: command, + User: n.user, + Privileged: n.privileged, + } + cr, err := n.dockerInterface.client.ContainerExecCreate(ctx, n.dockerInterface.Name, cfg) + if err != nil { + return 0, err + } + + r, err := n.dockerInterface.client.ContainerExecAttach(ctx, cr.ID, cfg) + if err != nil { + return 0, err + } + err = containerExecPipe(r, stdin, stdout, stderr) + if err != nil { + return 0, err + } + info, err := n.dockerInterface.client.ContainerExecInspect(ctx, cr.ID) + if err != nil { + return 0, err + } + return info.ExitCode, nil +} + +// Close closes the executor +func (*DockerContainerExecInterface) Close() error { + return nil +} + +func containerExecPipe(resp types.HijackedResponse, inStream io.Reader, outStream, errorStream io.Writer) error { + var err error + receiveStdout := make(chan error, 1) + if outStream != nil || errorStream != nil { + go func() { + _, err = stdcopy.StdCopy(outStream, errorStream, resp.Reader) + receiveStdout <- err + }() + } + + stdinDone := make(chan struct{}) + go func() { + if inStream != nil { + io.Copy(resp.Conn, inStream) + } + + resp.CloseWrite() + close(stdinDone) + }() + + select { + case err := <-receiveStdout: + if err != nil { + return err + } + case <-stdinDone: + if outStream != nil || errorStream != nil { + if err := <-receiveStdout; err != nil { + return err + } + } + } + + return nil +} diff --git a/tests/e2e/framework/pod_interface.go b/tests/e2e/framework/pod_interface.go new file mode 100644 index 000000000..907c61621 --- /dev/null +++ b/tests/e2e/framework/pod_interface.go @@ -0,0 +1,197 @@ +/* +Copyright 2017 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "fmt" + "io" + "time" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + remotecommandconsts "k8s.io/apimachinery/pkg/util/remotecommand" + "k8s.io/client-go/pkg/api/v1" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/client-go/util/exec" + "k8s.io/kubernetes/pkg/api" +) + +// PodInterface provides API to work with a pod +type PodInterface struct { + controller *Controller + hasService bool + + Pod *v1.Pod +} + +func newPodInterface(controller *Controller, pod *v1.Pod) *PodInterface { + return &PodInterface{ + controller: controller, + Pod: pod, + } +} + +// Create creates pod in the k8s +func (pi *PodInterface) Create() error { + updatedPod, err := pi.controller.client.Pods(pi.controller.Namespace()).Create(pi.Pod) + if err != nil { + return err + } + pi.Pod = updatedPod + return nil +} + +// Delete deletes the pod and associated service, which was earlier created by `controller.Run()` +func (pi *PodInterface) Delete() error { + if pi.hasService { + pi.controller.client.Services(pi.controller.Namespace()).Delete(pi.Pod.Name, nil) + } + return pi.controller.client.Pods(pi.controller.Namespace()).Delete(pi.Pod.Name, nil) +} + +// Wait waits for pod to start and checks that it doesn't fail immediately after that +func (pi *PodInterface) Wait(timing ...time.Duration) error { + timeout := time.Minute * 5 + pollPeriond := time.Second + consistencyPeriod := time.Second * 5 + if len(timing) > 0 { + timeout = timing[0] + } + if len(timing) > 1 { + pollPeriond = timing[1] + } + if len(timing) > 2 { + consistencyPeriod = timing[2] + } + + return waitForConsistentState(func() error { + podUpdated, err := pi.controller.client.Pods(pi.Pod.Namespace).Get(pi.Pod.Name, metav1.GetOptions{}) + if err != nil { + return err + } + pi.Pod = podUpdated + phase := v1.PodRunning + if podUpdated.Status.Phase != phase { + return fmt.Errorf("pod %s is not %s phase: %s", podUpdated.Name, phase, podUpdated.Status.Phase) + } + return nil + }, timeout, pollPeriond, consistencyPeriod) +} + +// WaitDestruction waits for the pod to be deleted +func (pi *PodInterface) WaitDestruction(timing ...time.Duration) error { + timeout := time.Minute * 5 + pollPeriond := time.Second + consistencyPeriod := time.Second * 5 + if len(timing) > 0 { + timeout = timing[0] + } + if len(timing) > 1 { + pollPeriond = timing[1] + } + if len(timing) > 2 { + consistencyPeriod = timing[2] + } + + return waitForConsistentState(func() error { + _, err := pi.controller.client.Pods(pi.Pod.Namespace).Get(pi.Pod.Name, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + return fmt.Errorf("pod %s was not deleted", pi.Pod.Name) + }, timeout, pollPeriond, consistencyPeriod) +} + +// Container returns interface to execute commands in one of pod's containers +func (pi *PodInterface) Container(name string) (Executor, error) { + if name == "" && len(pi.Pod.Spec.Containers) > 0 { + name = pi.Pod.Spec.Containers[0].Name + } + found := false + for _, c := range pi.Pod.Spec.Containers { + if c.Name == name { + found = true + break + } + } + if !found { + return nil, fmt.Errorf("container %s doesn't exist in pod %s in namespace %s", name, pi.Pod.Name, pi.Pod.Namespace) + } + return &containerInterface{ + podInterface: pi, + name: name, + }, nil +} + +// DinDNodeExecutor return DinD executor for node, where this pod is located +func (pi *PodInterface) DinDNodeExecutor() (Executor, error) { + return pi.controller.DinDNodeExecutor(pi.Pod.Spec.NodeName) +} + +type containerInterface struct { + podInterface *PodInterface + name string +} + +// Exec executes commands in one of containers in the pod +func (ci *containerInterface) Exec(command []string, stdin io.Reader, stdout, stderr io.Writer) (int, error) { + restClient := ci.podInterface.controller.client.RESTClient() + req := restClient.Post(). + Resource("pods"). + Name(ci.podInterface.Pod.Name). + Namespace(ci.podInterface.Pod.Namespace). + SubResource("exec") + req.VersionedParams(&api.PodExecOptions{ + Container: ci.name, + Command: command, + Stdin: stdin != nil, + Stdout: stdout != nil, + Stderr: stderr != nil, + }, api.ParameterCodec) + + executor, err := remotecommand.NewExecutor(ci.podInterface.controller.restConfig, "POST", req.URL()) + if err != nil { + return 0, err + } + + exitCode := 0 + options := remotecommand.StreamOptions{ + SupportedProtocols: remotecommandconsts.SupportedStreamingProtocols, + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + } + err = executor.Stream(options) + if err != nil { + if c, ok := err.(exec.CodeExitError); ok { + exitCode = c.Code + err = nil + } + } + if err != nil { + return 0, err + } + return exitCode, nil +} + +// Close closes the executor +func (*containerInterface) Close() error { + return nil +} diff --git a/tests/e2e/framework/ssh_interface.go b/tests/e2e/framework/ssh_interface.go new file mode 100644 index 000000000..011635ba8 --- /dev/null +++ b/tests/e2e/framework/ssh_interface.go @@ -0,0 +1,114 @@ +/* +Copyright 2017 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "fmt" + "io" + "net" + "os" + "strings" + + "golang.org/x/crypto/ssh" +) + +type sshInterface struct { + vmInterface *VMInterface + client *ssh.Client +} + +func newSSHInterface(vmInterface *VMInterface, user, secret string) (*sshInterface, error) { + var authMethod ssh.AuthMethod + key := trimBlock(secret) + signer, err := ssh.ParsePrivateKey([]byte(key)) + if err != nil { + authMethod = ssh.Password(secret) + } else { + authMethod = ssh.PublicKeys(signer) + } + + config := &ssh.ClientConfig{ + User: user, + Auth: []ssh.AuthMethod{authMethod}, + } + + vmPod, err := vmInterface.Pod() + if err != nil { + return nil, err + } + + virtletPod, err := vmInterface.VirtletPod() + if err != nil { + return nil, err + } + container, err := virtletPod.Container("virtlet") + if err != nil { + return nil, err + } + + client, server := net.Pipe() + go func() { + container.Exec([]string{"nc", "-q0", vmPod.Pod.Status.PodIP, "22"}, server, server, os.Stderr) + client.Close() + }() + + conn, chans, reqs, err := ssh.NewClientConn(client, fmt.Sprintf("%s.%s", vmPod.Pod.Name, vmPod.Pod.Namespace), config) + if err != nil { + return nil, err + } + + sshClient := ssh.NewClient(conn, chans, reqs) + return &sshInterface{ + vmInterface: vmInterface, + client: sshClient, + }, nil +} + +func (si *sshInterface) Exec(command []string, stdin io.Reader, stdout, stderr io.Writer) (int, error) { + session, err := si.client.NewSession() + if err != nil { + return 0, err + } + defer session.Close() + + session.Stdout = stdout + session.Stderr = stderr + session.Stdin = stdin + + exitcode := 0 + for i, arg := range command { + if i == 0 { + continue + } + command[i] = fmt.Sprintf("'%s'", strings.Replace(arg, "'", "\\'", -1)) + } + err = session.Run(strings.Join(command, " ")) + if err != nil { + if s, ok := err.(*ssh.ExitError); ok { + exitcode = s.ExitStatus() + err = nil + } + } + if err != nil { + return 0, err + } + return exitcode, nil +} + +func (si *sshInterface) Close() error { + return si.client.Close() +} diff --git a/tests/e2e/framework/vm_interface.go b/tests/e2e/framework/vm_interface.go new file mode 100644 index 000000000..9d3450ffe --- /dev/null +++ b/tests/e2e/framework/vm_interface.go @@ -0,0 +1,238 @@ +/* +Copyright 2017 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "encoding/xml" + "fmt" + "regexp" + "strconv" + "time" + + libvirtxml "github.com/libvirt/libvirt-go-xml" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/pkg/api/v1" +) + +// VMInterface provides API to work with virtlet VM pods +type VMInterface struct { + controller *Controller + pod *PodInterface + + Name string +} + +// VMOptions defines VM parameters +type VMOptions struct { + Image string + VCPUCount int + SSHKey string + CloudInitScript string + DiskDriver string + Limits map[string]string +} + +func newVMInterface(controller *Controller, name string) *VMInterface { + return &VMInterface{ + controller: controller, + Name: name, + } +} + +// Pod returns ensures that underlying is started and returns it +func (vmi *VMInterface) Pod() (*PodInterface, error) { + if vmi.pod == nil { + pod, err := vmi.controller.Pod(vmi.Name, "") + if err != nil { + return nil, err + } + vmi.pod = pod + } + if vmi.pod == nil { + return nil, fmt.Errorf("pod %s in namespace %s cannot be found", vmi.Name, vmi.controller.namespace.Name) + } + if vmi.pod.Pod.Status.Phase != v1.PodRunning { + err := vmi.pod.Wait() + if err != nil { + return nil, err + } + } + return vmi.pod, nil +} + +// Create create new virtlet VM pod in k8s +func (vmi *VMInterface) Create(options VMOptions, waitTimeout time.Duration, beforeCreate func(*PodInterface)) error { + pod := newPodInterface(vmi.controller, vmi.buildVMPod(options)) + if beforeCreate != nil { + beforeCreate(pod) + } + err := pod.Create() + if err != nil { + return err + } + err = pod.Wait(waitTimeout) + if err != nil { + return err + } + vmi.pod = pod + return nil +} + +// Delete deletes VM pod and waits for it to disappear from k8s +func (vmi *VMInterface) Delete(waitTimeout time.Duration) error { + if vmi.pod == nil { + return nil + } + vmi.pod.Delete() + return vmi.pod.WaitDestruction(waitTimeout) +} + +// VirtletPod returns pod in which virtlet instance, responsible for this VM is located +// (i.e. kube-system:virtlet-xxx pod on the same node) +func (vmi *VMInterface) VirtletPod() (*PodInterface, error) { + vmPod, err := vmi.Pod() + if err != nil { + return nil, err + } + + node := vmPod.Pod.Spec.NodeName + pod, err := vmi.controller.FindPod("kube-system", map[string]string{"runtime": "virtlet"}, + func(pod *PodInterface) bool { + return pod.Pod.Spec.NodeName == node + }, + ) + if err != nil { + return nil, err + } else if pod == nil { + return nil, fmt.Errorf("cannot find virtlet pod on node %s", node) + } + return pod, nil +} + +func (vmi *VMInterface) buildVMPod(options VMOptions) *v1.Pod { + annotations := map[string]string{ + "kubernetes.io/target-runtime": "virtlet", + "VirtletDiskDriver": options.DiskDriver, + "VirtletSSHKeys": options.SSHKey, + "VirtletCloudInitUserDataScript": options.SSHKey, + } + if options.VCPUCount > 0 { + annotations["VirtletVCPUCount"] = strconv.Itoa(options.VCPUCount) + } + + limits := v1.ResourceList{} + for k, v := range options.Limits { + limits[v1.ResourceName(k)] = resource.MustParse(v) + } + + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: vmi.Name, + Namespace: vmi.controller.namespace.Name, + Annotations: annotations, + }, + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + NodeAffinity: &v1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ + NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "extraRuntime", + Operator: "In", + Values: []string{"virtlet"}, + }, + }, + }, + }, + }, + }, + }, + Containers: []v1.Container{ + { + Name: vmi.Name, + Image: "virtlet/" + options.Image, + Resources: v1.ResourceRequirements{ + Limits: limits, + }, + }, + }, + }, + } +} + +// SSH returns SSH executor that can run commands in VM +func (vmi *VMInterface) SSH(user, secret string) (Executor, error) { + return newSSHInterface(vmi, user, secret) +} + +// DomainName returns libvirt domain name the VM +func (vmi *VMInterface) DomainName() (string, error) { + pod, err := vmi.Pod() + if err != nil { + return "", err + } + containerID := pod.Pod.Status.ContainerStatuses[0].ContainerID + match := regexp.MustCompile("__(.+)$").FindStringSubmatch(containerID) + if len(match) < 2 { + return "", fmt.Errorf("invalid container ID %q", containerID) + } + return fmt.Sprintf("virtlet-%s-%s", match[1][:13], pod.Pod.Status.ContainerStatuses[0].Name), nil +} + +// VirshCommand runs virsh command in the virtlet pod, responsible for this VM +// Domain name is automatically substituted into commandline in place of `` +func (vmi *VMInterface) VirshCommand(command ...string) (string, error) { + virtletPod, err := vmi.VirtletPod() + if err != nil { + return "", err + } + for i, c := range command { + switch c { + case "": + domainName, err := vmi.DomainName() + if err != nil { + return "", err + } + command[i] = domainName + } + } + return RunVirsh(virtletPod, command...) +} + +// Domain returns libvirt domain definition for the VM +func (vmi *VMInterface) Domain() (libvirtxml.Domain, error) { + domainXML, err := vmi.VirshCommand("dumpxml", "") + if err != nil { + return libvirtxml.Domain{}, err + } + var domain libvirtxml.Domain + err = xml.Unmarshal([]byte(domainXML), &domain) + return domain, err +} + +// RunVirsh runs virsh command in the given virtlet pod +func RunVirsh(virtletPod *PodInterface, command ...string) (string, error) { + container, err := virtletPod.Container("virtlet") + if err != nil { + return "", err + } + cmd := append([]string{"virsh"}, command...) + return ExecSimple(container, cmd...) +} diff --git a/tests/e2e/ginkgo-ext/matchers.go b/tests/e2e/ginkgo-ext/matchers.go new file mode 100644 index 000000000..67522ed05 --- /dev/null +++ b/tests/e2e/ginkgo-ext/matchers.go @@ -0,0 +1,40 @@ +/* +Copyright 2017 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ginkgoext + +import ( + "github.com/onsi/gomega/types" +) + +type anythingMatcher struct{} + +func (matcher *anythingMatcher) Match(actual interface{}) (success bool, err error) { + return true, nil +} + +func (matcher *anythingMatcher) FailureMessage(actual interface{}) (message string) { + return "" +} + +func (matcher *anythingMatcher) NegatedFailureMessage(actual interface{}) (message string) { + return "" +} + +// BeAnything returns matcher that matches any value +func BeAnything() types.GomegaMatcher { + return &anythingMatcher{} +} diff --git a/tests/e2e/ginkgo-ext/scopes.go b/tests/e2e/ginkgo-ext/scopes.go new file mode 100644 index 000000000..1cea0f7f4 --- /dev/null +++ b/tests/e2e/ginkgo-ext/scopes.go @@ -0,0 +1,239 @@ +/* +Copyright 2017 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ginkgoext + +import ( + "reflect" + "sync/atomic" + + "github.com/onsi/ginkgo" +) + +type scope struct { + parent *scope + children []*scope + counter int32 + before []func() + after []func() + started int32 + failed bool + normalTests int + focusedTests int + focused bool +} + +var ( + currentScope = &scope{} + rootScope = currentScope + countersInitialized bool + + Context = wrapContextFunc(ginkgo.Context, false) + FContext = wrapContextFunc(ginkgo.FContext, true) + PContext = wrapNilContextFunc(ginkgo.PContext) + XContext = wrapNilContextFunc(ginkgo.XContext) + Describe = wrapContextFunc(ginkgo.Describe, false) + FDescribe = wrapContextFunc(ginkgo.FDescribe, true) + PDescribe = wrapNilContextFunc(ginkgo.PDescribe) + XDescribe = wrapNilContextFunc(ginkgo.XDescribe) + It = wrapItFunc(ginkgo.It, false) + FIt = wrapItFunc(ginkgo.FIt, true) + PIt = ginkgo.PIt + XIt = ginkgo.XIt + By = ginkgo.By + JustBeforeEach = ginkgo.JustBeforeEach + BeforeSuite = ginkgo.BeforeSuite + AfterSuite = ginkgo.AfterSuite + Skip = ginkgo.Skip + Fail = ginkgo.Fail + CurrentGinkgoTestDescription = ginkgo.CurrentGinkgoTestDescription + GinkgoRecover = ginkgo.GinkgoRecover + GinkgoT = ginkgo.GinkgoT + RunSpecs = ginkgo.RunSpecs + RunSpecsWithCustomReporters = ginkgo.RunSpecsWithCustomReporters + RunSpecsWithDefaultAndCustomReporters = ginkgo.RunSpecsWithDefaultAndCustomReporters +) + +type Done ginkgo.Done + +// BeforeAll runs the function once before any test in context +func BeforeAll(body func()) bool { + if currentScope != nil { + if body == nil { + currentScope.before = nil + return true + } + currentScope.before = append(currentScope.before, body) + return BeforeEach(func() {}) + } + return true +} + +// AfterAll runs the function once after any test in context +func AfterAll(body func()) bool { + if currentScope != nil { + if body == nil { + currentScope.before = nil + return true + } + currentScope.after = append(currentScope.after, body) + return AfterEach(func() {}) + } + return true +} + +// BeforeEach runs the function before each test in context +func BeforeEach(body interface{}, timeout ...float64) bool { + if currentScope == nil { + return ginkgo.BeforeEach(body, timeout...) + } + cs := currentScope + before := func() { + if atomic.CompareAndSwapInt32(&cs.started, 0, 1) && cs.before != nil { + defer func() { + if r := recover(); r != nil { + cs.failed = true + panic(r) + } + }() + for _, before := range cs.before { + before() + } + } else if cs.failed { + Fail("failed due to BeforeAll failure") + } + } + return ginkgo.BeforeEach(applyAdvice(body, before, nil), timeout...) +} + +// AfterEach runs the function after each test in context +func AfterEach(body interface{}, timeout ...float64) bool { + if currentScope == nil { + return ginkgo.AfterEach(body, timeout...) + } + cs := currentScope + after := func() { + if cs.counter == 0 && cs.after != nil { + for _, after := range cs.after { + after() + } + } + } + return ginkgo.AfterEach(applyAdvice(body, nil, after), timeout...) +} + +func wrapContextFunc(fn func(string, func()) bool, focused bool) func(string, func()) bool { + return func(text string, body func()) bool { + if currentScope == nil { + return fn(text, body) + } + newScope := &scope{parent: currentScope, focused: focused} + currentScope.children = append(currentScope.children, newScope) + currentScope = newScope + res := fn(text, body) + currentScope = currentScope.parent + return res + } +} + +func wrapNilContextFunc(fn func(string, func()) bool) func(string, func()) bool { + return func(text string, body func()) bool { + oldScope := currentScope + currentScope = nil + res := fn(text, body) + currentScope = oldScope + return res + } +} + +func wrapItFunc(fn func(string, interface{}, ...float64) bool, focused bool) func(string, interface{}, ...float64) bool { + if !countersInitialized { + countersInitialized = true + BeforeSuite(func() { + calculateCounters(rootScope, false) + }) + } + return func(text string, body interface{}, timeout ...float64) bool { + if currentScope == nil { + return fn(text, body, timeout...) + } + if focused { + currentScope.focusedTests++ + } else { + currentScope.normalTests++ + } + return fn(text, wrapTest(body), timeout...) + } +} + +func applyAdvice(f interface{}, before, after func()) interface{} { + fn := reflect.ValueOf(f) + template := func(in []reflect.Value) []reflect.Value { + if before != nil { + before() + } + if after != nil { + defer after() + } + return fn.Call(in) + } + v := reflect.MakeFunc(fn.Type(), template) + return v.Interface() +} + +func wrapTest(f interface{}) interface{} { + cs := currentScope + after := func() { + for cs != nil { + atomic.AddInt32(&cs.counter, -1) + cs = cs.parent + } + } + return applyAdvice(f, nil, after) +} + +func calculateCounters(s *scope, focusedOnly bool) (int, bool) { + count := s.focusedTests + haveFocused := s.focusedTests > 0 + var focusedChildren int + for _, child := range s.children { + if child.focused { + c, _ := calculateCounters(child, false) + focusedChildren += c + } + } + if focusedChildren > 0 { + haveFocused = true + count += focusedChildren + } + var normalChildren int + for _, child := range s.children { + if !child.focused { + c, f := calculateCounters(child, focusedOnly || haveFocused) + if f { + haveFocused = true + count += c + } else { + normalChildren += c + } + } + } + if !focusedOnly && !haveFocused { + count += s.normalTests + normalChildren + } + s.counter = int32(count) + return count, haveFocused +} diff --git a/tests/e2e/hung_vm_test.go b/tests/e2e/hung_vm_test.go new file mode 100644 index 000000000..849347314 --- /dev/null +++ b/tests/e2e/hung_vm_test.go @@ -0,0 +1,61 @@ +/* +Copyright 2017 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "time" + + . "github.com/onsi/gomega" + + "github.com/Mirantis/virtlet/tests/e2e/framework" + . "github.com/Mirantis/virtlet/tests/e2e/ginkgo-ext" +) + +var _ = Describe("Hung VM", func() { + var ( + vm *framework.VMInterface + vmPod *framework.PodInterface + ssh framework.Executor + ) + + BeforeAll(func() { + vm = controller.VM("hung-vm") + vm.Create(framework.VMOptions{ + Image: *cirrosLocation, + SSHKey: sshPublicKey, + VCPUCount: 1, + DiskDriver: "virtio", + Limits: map[string]string{ + "memory": "128Mi", + }, + }, time.Minute*5, nil) + var err error + vmPod, err = vm.Pod() + Expect(err).NotTo(HaveOccurred()) + }) + + scheduleWaitSSH(&vm, &ssh) + + It("Must be successfully deleted after it hangs", func() { + Eventually(framework.WithTimeout(time.Second*2, func() error { + _, err := framework.ExecSimple(ssh, "sudo /sbin/halt -nf") + return err + })).Should(MatchError(framework.ErrTimeout)) + + deleteVM(vm) + }) +}) diff --git a/tests/e2e/resources_test.go b/tests/e2e/resources_test.go new file mode 100644 index 000000000..7f20c729f --- /dev/null +++ b/tests/e2e/resources_test.go @@ -0,0 +1,72 @@ +/* +Copyright 2017 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "regexp" + "strconv" + "time" + + . "github.com/onsi/gomega" + + "github.com/Mirantis/virtlet/tests/e2e/framework" + . "github.com/Mirantis/virtlet/tests/e2e/ginkgo-ext" +) + +var _ = Describe("VM resources", func() { + var ( + vm *framework.VMInterface + ssh framework.Executor + ) + + BeforeAll(func() { + vm = controller.VM("cirros-vm2") + vm.Create(framework.VMOptions{ + Image: *cirrosLocation, + SSHKey: sshPublicKey, + VCPUCount: 2, + DiskDriver: "virtio", + Limits: map[string]string{ + "memory": "128Mi", + "cpu": "500m", + }, + }, time.Minute*5, nil) + do(vm.Pod()) + }) + + AfterAll(func() { + deleteVM(vm) + }) + + scheduleWaitSSH(&vm, &ssh) + + It("Should have CPU count as set for the domain", func() { + checkCPUCount(vm, ssh, 2) + }) + + It("Should have total memory amount close to that set for the domain", func() { + meminfo := do(framework.ExecSimple(ssh, "cat", "/proc/meminfo")).(string) + totals := regexp.MustCompile(`(?:DirectMap4k|DirectMap2M):\s+(\d+)`).FindAllStringSubmatch(meminfo, -1) + Expect(totals).To(HaveLen(2)) + total := 0 + for _, m := range totals { + Expect(m).To(HaveLen(2)) + total += do(strconv.Atoi(m[1])).(int) + } + Expect(total).To(Equal(130944)) + }) +}) diff --git a/tests/e2e/run_ceph.sh b/tests/e2e/run_ceph.sh deleted file mode 100755 index fc5c1acf0..000000000 --- a/tests/e2e/run_ceph.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash -# Copyright 2017 Mirantis -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -o errexit -set -o nounset -set -o pipefail -set -o errtrace - -SCRIPT_DIR="${1}" -# Get default gateway ip from kube-master node deployed by kubeadm-dind tool -MON_IP=$(docker exec kube-master route | grep default | awk '{print $2}') -CEPH_PUBLIC_NETWORK=${MON_IP}/16 -container_name="ceph_cluster" - -docker rm -fv ${container_name} >&/dev/null || true - -docker run -d --net=host -e MON_IP=${MON_IP} -e CEPH_PUBLIC_NETWORK=${CEPH_PUBLIC_NETWORK} --name ${container_name} ceph/demo - -# Check cluster is running -set +e -ntries=20 -echo -e -n "\tWaiting for ceph cluster..." -while ! docker exec ${container_name} ceph -s 2> /dev/null 1> /dev/null; do - if [ $ntries -eq 0 ]; then - echo "Failed to get ceph cluster status. Cluster is not running." - exit 1 - fi - sleep 2 - ((ntries--)) - echo -n "." -done -echo "Cluster started!" -set -e - -# Adjust ceph configs -docker exec ${container_name} /bin/bash -c 'echo -e "rbd default features = 1\nrbd default format = 2" >> /etc/ceph/ceph.conf' - -# Add rbd pool and volume -docker exec ${container_name} ceph osd pool create libvirt-pool 8 8 -docker exec ceph_cluster /bin/bash -c "apt-get update && apt-get install -y qemu-utils" -docker exec ${container_name} qemu-img create -f rbd rbd:libvirt-pool/rbd-test-image1 10M -docker exec ${container_name} qemu-img create -f rbd rbd:libvirt-pool/rbd-test-image2 10M -docker exec ${container_name} qemu-img create -f rbd rbd:libvirt-pool/rbd-test-image-pv 10M - -# Add user for virtlet -docker exec ${container_name} ceph auth get-or-create client.libvirt -docker exec ceph_cluster ceph auth caps client.libvirt mon "allow *" osd "allow *" msd "allow *" -SECRET="$(docker exec ${container_name} ceph auth get-key client.libvirt)" - -# Put secret into definition -sed "s^@MON_IP@^${MON_IP}^g;s^@SECRET@^${SECRET}^g" \ - "${SCRIPT_DIR}/../../examples/cirros-vm-rbd-volume.yaml.tmpl" \ - > "${SCRIPT_DIR}/cirros-vm-rbd-volume.yaml" -sed "s^@MON_IP@^${MON_IP}^g;s^@SECRET@^${SECRET}^g" \ - "${SCRIPT_DIR}/../../examples/cirros-vm-rbd-pv-volume.yaml.tmpl" \ - > "${SCRIPT_DIR}/cirros-vm-rbd-pv-volume.yaml"