Fix some issues of auth job and lb job (apache#110)

- Improve the `sed` statement to avoid using specific line number. - Remove the healthcheck woorkaround since it has been fixed in upstream - Extend the time to wait LB deployment. - Do not need collect-k8s-logs role
bzhaoopenstack · Apr 2, 2018 · ab96980 · ab96980
1 parent 7ade7ce
commit ab96980
Show file tree

Hide file tree

Showing 6 changed files with 47 additions and 41 deletions.
diff --git a/.../cloud-provider-openstack-acceptance-test-keystone-authentication-authorization/post.yaml b/.../cloud-provider-openstack-acceptance-test-keystone-authentication-authorization/post.yaml
diff --git a/...s/cloud-provider-openstack-acceptance-test-keystone-authentication-authorization/run.yaml b/...s/cloud-provider-openstack-acceptance-test-keystone-authentication-authorization/run.yaml
@@ -96,8 +96,8 @@
           sudo rm -rf /var/lib/kubelet/*
 
           # location of where the kubernetes processes log their output
-          mkdir -p /opt/stack/logs/
-          export LOG_DIR=/opt/stack/logs
+          mkdir -p "{{ ansible_user_dir }}/workspace/logs/kubernetes"
+          export LOG_DIR="{{ ansible_user_dir }}/workspace/logs/kubernetes"
           # We need this for one of the conformance tests
           export ALLOW_PRIVILEGED=true
           # Just kick off all the processes and drop down to the command line
@@ -109,11 +109,9 @@
           pushd ${GOPATH}/src/k8s.io/kubernetes
           export AUTHORIZATION_MODE="Webhook,Node"
 
-          # TODO: Following is workaround for supporting keystone webhook in local-up-cluster.sh tool, it should be landed in the official kubernetes repo
-          sed 's/curl --max-time 1/curl --max-time 5/g' -i ./hack/lib/util.sh
-          sed '583,587 d' -i ./hack/local-up-cluster.sh
-          sed '555 a \      --authentication-token-webhook-config-file=/etc/kubernetes/webhook.kubeconfig \\'  -i ./hack/local-up-cluster.sh
-          sed '555 a \      --authorization-webhook-config-file=/etc/kubernetes/webhook.kubeconfig \\'  -i ./hack/local-up-cluster.sh
+          sed -i -e "/kube::util::wait_for_url.*$/,+1d" hack/local-up-cluster.sh
+          sed -i -e '/hyperkube\" apiserver.*$/a \      --authentication-token-webhook-config-file=/etc/kubernetes/webhook.kubeconfig \\' hack/local-up-cluster.sh
+          sed -i -e '/hyperkube\" apiserver.*$/a \      --authorization-webhook-config-file=/etc/kubernetes/webhook.kubeconfig \\' hack/local-up-cluster.sh
 
           # -E preserves the current env vars, but we need to special case PATH
           sudo -E PATH=$PATH SHELLOPTS=$SHELLOPTS ./hack/local-up-cluster.sh -O

diff --git a/playbooks/cloud-provider-openstack-acceptance-test-lb-octavia/post.yaml b/playbooks/cloud-provider-openstack-acceptance-test-lb-octavia/post.yaml
@@ -1,7 +1,5 @@
 - hosts: all
   become: yes
-  roles:
-    - collect-k8s-logs
   tasks:
     - name: Clean kubernetes deployments
       shell:
@@ -10,9 +8,29 @@
           set -x
           pushd ${GOPATH}/src/k8s.io/kubernetes
           cluster/kubectl.sh config use-context local
-          cluster/kubectl.sh delete -f "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/examples/loadbalancers/external-http-nginx.yaml" || true
+          { source "{{ ansible_user_dir }}/openrc";}  1> /dev/null 2>&1
+          ext_lb_svc_uid=$(cluster/kubectl.sh get services external-http-nginx-service -o=jsonpath='{.metadata.uid}') || true
+          int_lb_svc_uid=$(cluster/kubectl.sh get services internal-http-nginx-service -o=jsonpath='{.metadata.uid}') || true
+
           cluster/kubectl.sh delete -f "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/examples/loadbalancers/internal-http-nginx.yaml" || true
+          cluster/kubectl.sh delete -f "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/examples/loadbalancers/external-http-nginx.yaml" || true
           popd
+
+          for lb_svc_uid in $ext_lb_svc_uid $int_lb_svc_uid; do
+              lb_name=$(echo $lb_svc_uid | tr -d '-' | sed 's/^/a/' | cut -c -32)
+              {
+                  for pool in $(neutron lbaas-pool-list -f value -c id -c name|grep $lb_name | awk '{print $1}'); do
+                      neutron lbaas-pool-delete $pool
+                      sleep 2
+                  done
+                  for listener in $(neutron lbaas-listener-list -f value -c id -c name |grep $lb_name | awk '{print $1}'); do
+                      neutron lbaas-listener-delete $listener
+                      sleep 2
+                  done
+                  neutron lbaas-loadbalancer-delete $lb_name
+                  sleep 2
+              } || true
+          done
         executable: /bin/bash
         chdir: '{{ zuul.project.src_dir }}'
       environment: '{{ golang_env }}'
diff --git a/playbooks/cloud-provider-openstack-acceptance-test-lb-octavia/run.yaml b/playbooks/cloud-provider-openstack-acceptance-test-lb-octavia/run.yaml
@@ -9,8 +9,10 @@
         cmd: |
           set -e
           apt-get install python-pip -y
-          pip install -U python-openstackclient
+          pip install -U python-openstackclient python-octaviaclient python-neutronclient
 
+          cat << EOF >> "{{ ansible_user_dir }}/openrc"
+          export OS_DOMAIN_NAME=$(echo '{{ vexxhost_credentials.user_domain_name }}')
           export OS_AUTH_TYPE=$(echo '{{ vexxhost_credentials.auth_type }}')
           export OS_IDENTITY_API_VERSION=$(echo '{{ vexxhost_credentials.identity_api_version }}')
           export OS_VOLUME_API_VERSION=$(echo '{{ vexxhost_credentials.volume_api_version }}')
@@ -23,6 +25,8 @@
           export OS_USERNAME=$(echo '{{ vexxhost_credentials.username }}')
           export OS_PASSWORD=$(echo '{{ vexxhost_credentials.password }}')
           export OS_REGION_NAME=$(echo '{{ vexxhost_credentials.region_name }}')
+          EOF
+          { source "{{ ansible_user_dir }}/openrc";}  1> /dev/null 2>&1
 
           if [[ ! -d "/etc/kubernetes/" ]]; then
               sudo mkdir -p /etc/kubernetes/
@@ -79,8 +83,8 @@
           sudo rm -rf /var/lib/kubelet/*
 
           # location of where the kubernetes processes log their output
-          mkdir -p /opt/stack/logs/
-          export LOG_DIR=/opt/stack/logs
+          mkdir -p "{{ ansible_user_dir }}/workspace/logs/kubernetes"
+          export LOG_DIR="{{ ansible_user_dir }}/workspace/logs/kubernetes"
           # We need this for one of the conformance tests
           export ALLOW_PRIVILEGED=true
           # Just kick off all the processes and drop down to the command line
@@ -90,8 +94,7 @@
 
           pushd ${GOPATH}/src/k8s.io/kubernetes
 
-          # TODO: this is a workaround until https://github.com/kubernetes/kubernetes/pull/61401 landed.
-          sed 's/curl --max-time 1/curl --max-time 5/g' -i ./hack/lib/util.sh
+          export MAX_TIME_FOR_URL_API_SERVER=5
           # -E preserves the current env vars, but we need to special case PATH
           sudo -E PATH=$PATH SHELLOPTS=$SHELLOPTS ./hack/local-up-cluster.sh -O
 
@@ -115,24 +118,26 @@
 
           cluster/kubectl.sh create -f "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/examples/loadbalancers/external-http-nginx.yaml"
           cluster/kubectl.sh create -f "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}/examples/loadbalancers/internal-http-nginx.yaml"
-          cluster/kubectl.sh get pods
-          cluster/kubectl.sh get services
 
-          for i in $(seq 1 60); do
-              external_lb_ip=$(./cluster/kubectl.sh describe services external-http-nginx-service |grep "LoadBalancer Ingress" | awk -F ' ' '{print $3}')
-              internal_lb_ip=$(./cluster/kubectl.sh describe services internal-http-nginx-service |grep "LoadBalancer Ingress" | awk -F ' ' '{print $3}')
+          for i in $(seq 1 120); do
+              if [ -z "${external_lb_ip}" ]; then
+                  external_lb_ip=$(./cluster/kubectl.sh describe services external-http-nginx-service |grep "LoadBalancer Ingress" | awk -F ' ' '{print $3}')
+              fi
+              if [ -z "${internal_lb_ip}" ]; then
+                  internal_lb_ip=$(./cluster/kubectl.sh describe services internal-http-nginx-service |grep "LoadBalancer Ingress" | awk -F ' ' '{print $3}')
+              fi
+
               if [ -n "${external_lb_ip}" -a -n "${internal_lb_ip}" ]; then
                   break
               fi
-              echo "Waiting for deploying external-http-nginx-service service on try ${i} ..."
+              echo "Waiting for deploying loadbalancer service on try ${i} ..."
               sleep 5
           done
+
           if [ -z "${external_lb_ip}" -o -z "${internal_lb_ip}" ]; then
-              echo "Timed out to wait for external-http-nginx-service and internal-http-nginx-service services deployment!"
-              cluster/kubectl.sh describe services internal-http-nginx-service
-              cluster/kubectl.sh describe services external-http-nginx-service
-              cluster/kubectl.sh get pods
-              cluster/kubectl.sh get services
+              echo "Timed out to wait for loadbalancer services deployment!"
+              cluster/kubectl.sh describe pods
+              cluster/kubectl.sh describe services
               exit 1
           fi
 

diff --git a/roles/collect-k8s-logs/tasks/main.yaml b/roles/collect-k8s-logs/tasks/main.yaml
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
@@ -254,7 +254,6 @@
       Run acceptance tests of cloud-provider-openstack repo of K8S+keystone authentication and
       authorization scenario against vexxhost cloud
     run: playbooks/cloud-provider-openstack-acceptance-test-keystone-authentication-authorization/run.yaml
-    post-run: playbooks/cloud-provider-openstack-acceptance-test-keystone-authentication-authorization/post.yaml
     secrets:
       - vexxhost_credentials
     nodeset: ubuntu-xenial-vexxhost