diff --git a/.github/workflows/image-pull-and-tag.yaml b/.github/workflows/image-pull-and-tag.yaml index fa9afab2..7727298b 100644 --- a/.github/workflows/image-pull-and-tag.yaml +++ b/.github/workflows/image-pull-and-tag.yaml @@ -1,4 +1,4 @@ -name: Migrate Images to GHCR +name: Migrate Images to QUAY on: push: diff --git a/.github/workflows/testing-deploy-openstack.yaml b/.github/workflows/testing-deploy-openstack.yaml new file mode 100644 index 00000000..4994f862 --- /dev/null +++ b/.github/workflows/testing-deploy-openstack.yaml @@ -0,0 +1,45 @@ +name: testing-openstack-deploy + +on: + workflow_run: + workflows: + - Migrate Images to QUAY + types: + - completed + workflow_dispatch: + +jobs: + deploy: + runs-on: ubuntu-22.04 + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Python environment + run: sudo apt-get update && sudo apt-get install -y python3-pip + + - name: Install requirements + run: pip3 install -r testing/requirements.txt + + - name: Create OpenStack config directory + run: mkdir -p ~/.config/openstack + + - name: Retrieve clouds.yaml from GitHub secrets + env: + CLOUDS_YAML: ${{ secrets.CLOUDS_YAML }} + run: | + echo "$CLOUDS_YAML" > ~/.config/openstack/clouds.yaml + + - name: Retrieve env.yaml from GitHub secrets + env: + ENV_YAML: ${{ secrets.ENV_YAML }} + run: | + echo "$ENV_YAML" > ~/env.yaml + + - name: Run deployment script + run: bash testing/doit.sh + + - name: Cleanup + if: always() + run: bash testing/cleanup.sh diff --git a/testing/build.yaml b/testing/build.yaml new file mode 100644 index 00000000..3e339f94 --- /dev/null +++ b/testing/build.yaml @@ -0,0 +1,98 @@ +heat_template_version: wallaby + +description: Build genestack for testing + +parameters: + in_net: + type: string + description: Internal network name + + ext_net: + type: string + description: External network for floating IP + + image_name: + type: string + description: Image name for the VMs + + key_pair: + type: string + description: Key pair name for the VMs + default: "genestack-keypair" + + flavor_name: + type: string + description: Flavor name for the VMs + + security_group: + type: string + description: Security group for the VMs + + name1: + type: string + description: Name of first machine + default: "controller1" + + name2: + type: string + description: Name of second machine + default: "controller2" + + name3: + type: string + description: Name of third machine + default: "controller3" + +resources: + genestack_keypair: + type: OS::Nova::KeyPair + properties: + name: "genestack-keypair" + public_key: { get_file: ./key.pub } + + genestack_machine1: + type: OS::Heat::Stack + properties: + template: { get_file: single.yaml } + parameters: + internal: {get_param: in_net } + external: {get_param: ext_net } + image: {get_param: image_name } + flavor: {get_param: flavor_name } + keys: { get_resource: genestack_keypair } + sec_group: {get_param: security_group } + name: {get_param: name1 } + + genestack_machine2: + type: OS::Heat::Stack + properties: + template: { get_file: single.yaml } + parameters: + internal: {get_param: in_net } + external: {get_param: ext_net } + image: {get_param: image_name } + flavor: {get_param: flavor_name } + keys: { get_resource: genestack_keypair } + sec_group: {get_param: security_group } + name: {get_param: name2 } + + genestack_machine3: + type: OS::Heat::Stack + properties: + template: { get_file: single.yaml } + parameters: + internal: {get_param: in_net } + external: {get_param: ext_net } + image: {get_param: image_name } + flavor: {get_param: flavor_name } + keys: { get_resource: genestack_keypair } + sec_group: {get_param: security_group } + name: {get_param: name3 } + +outputs: + all_out: + description: ip addresses of the 3 machines + value: + - { get_attr: [genestack_machine1, outputs, vm_ip] } + - { get_attr: [genestack_machine2, outputs, vm_ip] } + - { get_attr: [genestack_machine3, outputs, vm_ip] } diff --git a/testing/cleanup.sh b/testing/cleanup.sh new file mode 100755 index 00000000..bc869542 --- /dev/null +++ b/testing/cleanup.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cd testing +rm ./key* +rm ./inventory.yaml +yes y | openstack --os-cloud default stack delete testing diff --git a/testing/deploy.yaml b/testing/deploy.yaml new file mode 100644 index 00000000..0f1de85e --- /dev/null +++ b/testing/deploy.yaml @@ -0,0 +1,352 @@ +--- +- name: Deploy genestack + hosts: "{{ groups['all'][0] }}" + vars: + ansible_user: root + tasks: + - name: Clone the Genestack repository + ansible.builtin.git: + repo: "https://github.com/rackerlabs/genestack" + dest: /opt/genestack + clone: yes + update: yes + recursive: yes + version: HEAD + + - name: botstrap cluster + shell: /opt/genestack/bootstrap.sh + + - name: Copy over inventory file + ansible.builtin.copy: + src: ./inventory.yaml + dest: /etc/genestack/inventory/inventory.yaml + owner: root + group: root + mode: '0644' + + - name: run setup-hosts.yaml + ansible.builtin.shell: | + source /opt/genestack/scripts/genestack.rc + cd /opt/genestack/ansible/playbooks + ansible-playbook host-setup.yml + args: + executable: /bin/bash + + - name: run cluster.yaml + ansible.builtin.shell: | + source /opt/genestack/scripts/genestack.rc + cd /opt/genestack/submodules/kubespray + ansible-playbook cluster.yml + args: + executable: /bin/bash + + - name: Label all nodes with specified labels + command: > + kubectl label nodes --all + role=storage-node + openstack-control-plane=enabled + openstack-compute-node=enabled + openstack-network-node=enabled + openstack-storage-node=enabled + longhorn.io/storage-node=enabled + node-role.kubernetes.io/worker=worker + --overwrite + + - name: Annotate nodes for OVN + command: > + kubectl annotate nodes --all \ + ovn.openstack.org/int_bridge='br-int' \ + ovn.openstack.org/bridges='br-ex' \ + ovn.openstack.org/ports='br-ex:fake1' \ + ovn.openstack.org/mappings='physnet1:br-ex' \ + ovn.openstack.org/availability_zones='az1' \ + ovn.openstack.org/gateway='enabled' + + - name: Remove taint + command: > + kubectl taint nodes --all node-role.kubernetes.io/control-plane:NoSchedule- + ignore_errors: yes + + - name: Install Prometheus + command: > + /opt/genestack/bin/install-prometheus.sh + + - name: Build openstack-helm + ansible.builtin.shell: | + cd /opt/genestack/submodules/openstack-helm + make all + + - name: Build openstack-helm-infra + ansible.builtin.shell: | + cd /opt/genestack/submodules/openstack-helm-infra + make all + +- name: Update package lists and install open-iscsi and cryptsetup + hosts: all + become: yes + tasks: + - name: Update apt package lists + apt: + update_cache: yes + + - name: Install open-iscsi and cryptsetup + apt: + name: + - open-iscsi + - cryptsetup + state: present + force_apt_get: yes + + - name: Load the dummy kernel module + ansible.builtin.command: + cmd: modprobe dummy + become: yes + + - name: Create a virtual interface fake1 + ansible.builtin.command: + cmd: ip link add fake1 type dummy + become: yes + + - name: Install pip for Python 3 + ansible.builtin.package: + name: python3-pip + state: present + + - name: Install Kubernetes Python client library + ansible.builtin.pip: + name: kubernetes + +- name: Continue deploying genestack + hosts: "{{ groups['all'][0] }}" + vars: + ansible_user: root + tasks: + - name: Add Longhorn Helm repository + command: helm repo add longhorn https://charts.longhorn.io + register: add_repo_result + changed_when: "'already exists' not in add_repo_result.stdout" + + - name: Update Helm repositories + command: helm repo update + register: update_repo_result + + - name: Display the result of repository addition + debug: + var: add_repo_result.stdout + + - name: Display the result of repository update + debug: + var: update_repo_result.stdout + + - name: Ensure the manifests exists + file: + path: /etc/genestack/manifests + state: directory + mode: '0755' + + - name: Write Longhorn configuration file + copy: + dest: /etc/genestack/manifests/longhorn.yaml + content: | + longhornManager: + nodeSelector: + longhorn.io/storage-node: "enabled" + longhornDriver: + nodeSelector: + longhorn.io/storage-node: "enabled" + longhornUI: + nodeSelector: + longhorn.io/storage-node: "enabled" + longhornConversionWebhook: + nodeSelector: + longhorn.io/storage-node: "enabled" + longhornAdmissionWebhook: + nodeSelector: + longhorn.io/storage-node: "enabled" + longhornRecoveryBackend: + nodeSelector: + longhorn.io/storage-node: "enabled" + mode: '0644' + + - name: Create namespace longhorn-system + command: kubectl create namespace longhorn-system + register: create_namespace_result + changed_when: "'already exists' not in create_namespace_result.stderr" + + - name: Label the namespace longhorn-system + command: > + kubectl label --overwrite namespace longhorn-system + pod-security.kubernetes.io/enforce=privileged + pod-security.kubernetes.io/enforce-version=latest + pod-security.kubernetes.io/warn=privileged + pod-security.kubernetes.io/warn-version=latest + pod-security.kubernetes.io/audit=privileged + pod-security.kubernetes.io/audit-version=latest + + - name: Install or upgrade Longhorn using Helm + command: helm upgrade --install longhorn longhorn/longhorn --namespace longhorn-system --create-namespace --version 1.7.2 -f /etc/genestack/manifests/longhorn.yaml + + - name: Apply the general StorageClass + k8s: + state: present + definition: + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: general + provisioner: driver.longhorn.io + allowVolumeExpansion: true + reclaimPolicy: Delete + volumeBindingMode: Immediate + parameters: + numberOfReplicas: "1" + dataLocality: "best-effort" + staleReplicaTimeout: "2880" + fromBackup: "" + fsType: "ext4" + + - name: Apply the general-multi-attach StorageClass + k8s: + state: present + definition: + kind: StorageClass + apiVersion: storage.k8s.io/v1 + metadata: + name: general-multi-attach + provisioner: driver.longhorn.io + allowVolumeExpansion: true + reclaimPolicy: Delete + volumeBindingMode: Immediate + parameters: + numberOfReplicas: "1" + dataLocality: "best-effort" + staleReplicaTimeout: "2880" + fromBackup: "" + fsType: "ext4" + accessMode: ReadWriteMany + + - name: Create Openstack namespace + command: kubectl apply -k /etc/genestack/kustomize/openstack + + - name: Create Openstack secrets + command: /opt/genestack/bin/create-secrets.sh + + - name: Apply secrets + command: kubectl create -f /etc/genestack/kubesecrets.yaml -n openstack + + - name: Apply metallb manifest + command: kubectl apply -f /etc/genestack/manifests/metallb/metallb-openstack-service-lb.yml + + - name: Create gateway namespace + command: kubectl create ns nginx-gateway + + - name: Install gateway CRDs + shell: | + kubectl kustomize "https://github.com/nginxinc/nginx-gateway-fabric/config/crd/gateway-api/standard?ref=v1.4.0" | kubectl apply -f - + args: + executable: /bin/bash + + - name: Install gateway controller + shell: | + cd /opt/genestack/submodules/nginx-gateway-fabric/charts + helm upgrade --install nginx-gateway-fabric ./nginx-gateway-fabric \ + --namespace=nginx-gateway \ + -f /etc/genestack/helm-configs/nginx-gateway-fabric/helm-overrides.yaml + args: + executable: /bin/bash + + - name: Restart cert manager + command: kubectl rollout restart deployment cert-manager --namespace cert-manager + + - name: Create gateway shared resource + shell: kubectl kustomize /etc/genestack/kustomize/gateway/nginx-gateway-fabric | kubectl apply -f - + args: + executable: /bin/bash + + - name: Deploy mariadb operator + shell: | + export cluster_name="cluster.local" + /opt/genestack/bin/install-mariadb-operator.sh + args: + executable: /bin/bash + + - name: Wait mariadb operator to be ready + command: kubectl wait --for=condition=available -n mariadb-system deployment mariadb-operator-webhook --timeout=300s + register: result + retries: 5 + delay: 10 + until: result.rc == 0 + + - name: Deploy mariadb cluster + command: kubectl --namespace openstack apply -k /etc/genestack/kustomize/mariadb-cluster/overlay + + - name: Deploy rabbitmq + shell: | + kubectl apply -k /etc/genestack/kustomize/rabbitmq-operator + kubectl apply -k /etc/genestack/kustomize/rabbitmq-topology-operator + kubectl apply -k /etc/genestack/kustomize/rabbitmq-cluster/overlay + + - name: Deploy memcached + command: /opt/genestack/bin/install-memcached.sh + + - name: Deploy libvirt + command: /opt/genestack/bin/install-libvirt.sh + + - name: Deploy OVN + command: kubectl apply -k /etc/genestack/kustomize/ovn + + - name: Wait 5 minutes for things to deploy + ansible.builtin.pause: + seconds: 300 + + - name: Install keystone + command: /opt/genestack/bin/install-keystone.sh + + - name: Install admin pod + command: kubectl --namespace openstack apply -f /etc/genestack/manifests/utils/utils-openstack-client-admin.yaml + + - name: Install glance + command: /opt/genestack/bin/install-glance.sh + + - name: Install heat + command: /opt/genestack/bin/install-heat.sh + + - name: Install barbican + command: /opt/genestack/bin/install-barbican.sh + + - name: Install cinder + command: /opt/genestack/bin/install-cinder.sh + + - name: Install placement + command: /opt/genestack/bin/install-placement.sh + + - name: Install nova + command: /opt/genestack/bin/install-nova.sh + + - name: Install neutron + command: /opt/genestack/bin/install-neutron.sh + + - name: Install skyline + command: kubectl --namespace openstack apply -k /etc/genestack/kustomize/skyline/overlay + + - name: Install octavia + command: /opt/genestack/bin/install-octavia.sh + + - name: Install magnum + command: /opt/genestack/bin/install-magnum.sh + + - name: Wait 10 minutes for things to deploy + ansible.builtin.pause: + seconds: 600 + + - name: Get status of all pods in openstack namespace + command: kubectl get pods --namespace openstack + + - name: Get status of compute services + command: kubectl exec -itn openstack openstack-admin-client -- openstack compute service list + + - name: Get status of network agents + command: kubectl exec -itn openstack openstack-admin-client -- openstack network agent list + + - name: Get status of volume services + command: kubectl exec -itn openstack openstack-admin-client -- openstack volume service list diff --git a/testing/doit.sh b/testing/doit.sh new file mode 100755 index 00000000..8ae58c3a --- /dev/null +++ b/testing/doit.sh @@ -0,0 +1,82 @@ +#!/bin/bash +cd testing +yes y | ssh-keygen -q -f ./key -N "" +openstack --os-cloud default stack create --wait -t build.yaml --environment ~/env.yaml testing +nodes=$(openstack --os-cloud default stack output show testing all_out -f value -c output_value) + +clean_list="${nodes//[\[\]\'\,]/}" # 'Remove brackets, single quotes, and commas +ips=($clean_list) # Convert the cleaned string into a Bash array + +# Define the machine names +machine_names=("controller1" "controller2" "controller3") + +# Initialize the YAML content +yaml_content="all:\n hosts:" + +# Generate the 'hosts' section using the Bash array +for i in "${!machine_names[@]}"; do + yaml_content+="\n ${machine_names[$i]}.openstacklocal:\n ansible_host: '${ips[$i]}'" +done + +# Add the 'children' section +yaml_content+="\n children:" +yaml_content+="\n k8s_cluster:" +yaml_content+="\n vars:" +yaml_content+="\n cluster_name: cluster.local" +yaml_content+="\n kube_ovn_iface: ens3" +yaml_content+="\n kube_ovn_default_interface_name: ens3" +yaml_content+="\n kube_ovn_central_hosts: \"{{ groups['ovn_network_nodes'] }}\"" +yaml_content+="\n children:" +yaml_content+="\n kube_control_plane:" +yaml_content+="\n hosts:" +for name in "${machine_names[@]}"; do + yaml_content+="\n ${name}.openstacklocal: null" +done + +yaml_content+="\n etcd:" +yaml_content+="\n hosts:" +for name in "${machine_names[@]}"; do + yaml_content+="\n ${name}.openstacklocal: null" +done + +yaml_content+="\n kube_node:" +yaml_content+="\n hosts:" +for name in "${machine_names[@]}"; do + yaml_content+="\n ${name}.openstacklocal: null" +done + +yaml_content+="\n openstack-control-plane:" +yaml_content+="\n hosts:" +for name in "${machine_names[@]}"; do + yaml_content+="\n ${name}.openstacklocal: null" +done + +yaml_content+="\n ovn_network_nodes:" +yaml_content+="\n hosts:" +for name in "${machine_names[@]}"; do + yaml_content+="\n ${name}.openstacklocal: null" +done + +yaml_content+="\n storage_nodes:" +yaml_content+="\n children:" +yaml_content+="\n ceph_storage_nodes:" +yaml_content+="\n hosts:" +for name in "${machine_names[@]}"; do + yaml_content+="\n ${name}.openstacklocal: null" +done + +yaml_content+="\n cinder_storage_nodes:" +yaml_content+="\n hosts: {}" +yaml_content+="\n openstack-compute-node:" +yaml_content+="\n hosts:" +for name in "${machine_names[@]}"; do + yaml_content+="\n ${name}.openstacklocal: null" +done + +echo -e "$yaml_content" > ./inventory.yaml + +sleep 5m + +# Prep all nodes +ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook -i ./inventory.yaml --private-key ./key fix-root.yaml +ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook -i ./inventory.yaml --private-key ./key deploy.yaml diff --git a/testing/env.yaml.example b/testing/env.yaml.example new file mode 100644 index 00000000..790c1664 --- /dev/null +++ b/testing/env.yaml.example @@ -0,0 +1,6 @@ +parameters: + in_net: INTERNAL_NETWORK + ext_net: EXTERNAL_NETWORK + image_name: jammy + flavor_name: genestack-build + security_group: security_group diff --git a/testing/fix-root.yaml b/testing/fix-root.yaml new file mode 100644 index 00000000..e0b3f431 --- /dev/null +++ b/testing/fix-root.yaml @@ -0,0 +1,28 @@ +--- +- name: Remove first 164 characters from /root/.ssh/authorized_keys + hosts: all + become: yes + vars: + ansible_user: ubuntu + tasks: + - name: Remove the first 164 characters from the authorized_keys file + shell: "tail -c +165 /root/.ssh/authorized_keys > /root/.ssh/authorized_keys.tmp && mv /root/.ssh/authorized_keys.tmp /root/.ssh/authorized_keys" + args: + executable: /bin/bash + become: yes + + - name: Ensure correct permissions on the authorized_keys file + file: + path: /root/.ssh/authorized_keys + owner: root + group: root + mode: '0600' + + - name: Copy ssh key to nodes + ansible.builtin.copy: + src: ./key + dest: /root/.ssh/id_rsa + owner: root + group: root + mode: '0600' + diff --git a/testing/requirements.txt b/testing/requirements.txt new file mode 100644 index 00000000..7dbba8a9 --- /dev/null +++ b/testing/requirements.txt @@ -0,0 +1,3 @@ +python-openstackclient +python-heatclient +ansible diff --git a/testing/single.yaml b/testing/single.yaml new file mode 100644 index 00000000..92036438 --- /dev/null +++ b/testing/single.yaml @@ -0,0 +1,63 @@ +heat_template_version: wallaby + +description: Deploy a single machine and floating ip + +parameters: + internal: + type: string + description: Internal network name + + external: + type: string + description: External network for floating IP + + image: + type: string + description: Image name for the VMs + + flavor: + type: string + description: Flavor name for the VMs + + keys: + type: string + description: Key pair name for the VMs + + sec_group: + type: string + description: Security group for the VMs + + name: + type: string + description: Name of first machine +resources: + # Create a Neutron port on the internal network + vm_port: + type: OS::Neutron::Port + properties: + network: { get_param: internal } + security_groups: + - { get_param: sec_group } + + # Allocate a floating IP on the external network and associate it with the VM's port + vm_floating_ip: + type: OS::Neutron::FloatingIP + properties: + floating_network: { get_param: external } + port_id: { get_resource: vm_port } + + # Create the VM with the specified image, flavor, key, and security group + vm: + type: OS::Nova::Server + properties: + name: { get_param: name } + image: { get_param: image } + flavor: { get_param: flavor } + key_name: { get_param: keys } + networks: + - port: { get_resource: vm_port } + +outputs: + vm_ip: + description: Floating IP address of the VM + value: { get_attr: [vm_floating_ip, floating_ip_address] }