From 3f9fbe4e00f7f2ad746b70a2d966f18989262ae4 Mon Sep 17 00:00:00 2001 From: hornjo Date: Sat, 16 Nov 2024 10:03:50 +0100 Subject: [PATCH] WIP --- .github/workflows/ansible-galaxy-publish.yml | 14 ++ README.md | 41 +++++- defaults/main.yml | 11 ++ handlers/main.yml | 1 + meta/main.yml | 34 +++++ tasks/bootstrap-rke.yml | 126 ++++++++++++++++++ tasks/join-servers.yml | 61 +++++++++ tasks/kube-vip.yml | 27 ++++ tasks/main.yml | 33 +++++ tasks/prepare-nodes.yml | 36 +++++ templates/kube-vip-controller.yaml.j2 | 88 +++++++++++++ templates/kube-vip.yaml.j2 | 130 +++++++++++++++++++ templates/rke2-agent.service.j2 | 13 ++ templates/rke2-server-config.j2 | 14 ++ templates/rke2-server.service.j2 | 13 ++ tests/ansible.cfg | 4 + tests/inventory.yml | 19 +++ tests/test.yml | 10 ++ 18 files changed, 674 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ansible-galaxy-publish.yml create mode 100644 defaults/main.yml create mode 100644 handlers/main.yml create mode 100644 meta/main.yml create mode 100644 tasks/bootstrap-rke.yml create mode 100644 tasks/join-servers.yml create mode 100644 tasks/kube-vip.yml create mode 100644 tasks/main.yml create mode 100644 tasks/prepare-nodes.yml create mode 100644 templates/kube-vip-controller.yaml.j2 create mode 100644 templates/kube-vip.yaml.j2 create mode 100644 templates/rke2-agent.service.j2 create mode 100644 templates/rke2-server-config.j2 create mode 100644 templates/rke2-server.service.j2 create mode 100644 tests/ansible.cfg create mode 100644 tests/inventory.yml create mode 100644 tests/test.yml diff --git a/.github/workflows/ansible-galaxy-publish.yml b/.github/workflows/ansible-galaxy-publish.yml new file mode 100644 index 0000000..f1e4ef4 --- /dev/null +++ b/.github/workflows/ansible-galaxy-publish.yml @@ -0,0 +1,14 @@ +name: anible-galaxy-publish +on: + push: + branches: ["main"] +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@v3 + - name: Ansible role Publish + uses: 0x022b/galaxy-role-import-action@1.0.0 + with: + galaxy_api_key: ${{ secrets.galaxy_api_key }} diff --git a/README.md b/README.md index acd9995..77158c5 100644 --- a/README.md +++ b/README.md @@ -1 +1,40 @@ -# rke2-installation \ No newline at end of file +# rke2-installation + +Role Name +========= + +This Role installes a HA RKE2 Cluster with the Kube-VIP for HA and the Loadbalancer to expose. As this is onpremise, the kube-vip-cloud-controler is used to expose the loadbalancer IPs. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/defaults/main.yml b/defaults/main.yml new file mode 100644 index 0000000..23aa9cc --- /dev/null +++ b/defaults/main.yml @@ -0,0 +1,11 @@ +--- +# RKE installation details +rke2_version: "" +arch: "" # amd64 or arm64 +rke2_install_dir: "/usr/local/bin" +rke2_binary_url: "https://github.com/rancher/rke2/releases/download/{{ rke2_version }}/rke2.linux-{{ arch }}" + +# Kube-VIP - for HA and Loadbalancer +kube_vip_version: v0.8.6 +vip_interface: eth0 +vip: "" diff --git a/handlers/main.yml b/handlers/main.yml new file mode 100644 index 0000000..ed97d53 --- /dev/null +++ b/handlers/main.yml @@ -0,0 +1 @@ +--- diff --git a/meta/main.yml b/meta/main.yml new file mode 100644 index 0000000..ea68190 --- /dev/null +++ b/meta/main.yml @@ -0,0 +1,34 @@ +galaxy_info: + author: your name + description: your role description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) + + min_ansible_version: 2.1 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. diff --git a/tasks/bootstrap-rke.yml b/tasks/bootstrap-rke.yml new file mode 100644 index 0000000..2933be7 --- /dev/null +++ b/tasks/bootstrap-rke.yml @@ -0,0 +1,126 @@ +- name: Create directory for RKE2 config + ansible.builtin.file: + path: "/etc/rancher/rke2" + state: directory + mode: '0644' + become: true + +- name: Create directory for RKE2 token + ansible.builtin.file: + path: "/var/lib/rancher/rke2/server" + state: directory + mode: '0644' + become: true + +- name: Deploy RKE2 server Configuration + ansible.builtin.template: + src: templates/rke2-server-config.j2 + dest: /etc/rancher/rke2/config.yaml + owner: root + group: root + mode: '0644' + when: inventory_hostname == groups["server"][0] + become: true + +- name: Create systemd service file for RKE2 server + ansible.builtin.template: + src: templates/rke2-server.service.j2 + dest: /etc/systemd/system/rke2-server.service + owner: root + group: root + mode: '0644' + when: inventory_hostname == groups["server"][0] + become: true + +- name: Reload RKE-Server service + ansible.builtin.systemd: + name: rke2-server + enabled: true + state: restarted + daemon_reload: true + when: inventory_hostname == groups["server"][0] + become: true + + +- name: Wait for node-token + ansible.builtin.wait_for: + path: /var/lib/rancher/rke2/server/node-token + when: inventory_hostname == groups["server"][0] + become: true + +- name: Wait for kubectl + ansible.builtin.wait_for: + path: /var/lib/rancher/rke2/bin/kubectl + when: inventory_hostname == groups["server"][0] + become: true + +- name: Copy kubectl to user bin + ansible.builtin.copy: + src: /var/lib/rancher/rke2/bin/kubectl + dest: /usr/local/bin/kubectl + mode: '0755' + remote_src: true + become: true + when: inventory_hostname == groups["server"][0] + +- name: Wait for kubectl + ansible.builtin.wait_for: + path: /usr/local/bin/kubectl + when: inventory_hostname == groups["server"][0] + +- name: Check if the token exists and get details + ansible.builtin.stat: + path: /var/lib/rancher/rke2/server + register: token_stats + become: true + +- name: Change file access for node-token + ansible.builtin.file: + path: /var/lib/rancher/rke2/server + mode: "g+rx,o+rx" + when: inventory_hostname == groups["server"][0] + become: true + +- name: Fetch the token from the first server node and make var accessible for all + ansible.builtin.slurp: + src: /var/lib/rancher/rke2/server/token + register: rke2_token + when: inventory_hostname == groups["server"][0] + run_once: true + become: true + +- name: Save Master node-token for later + ansible.builtin.set_fact: + token: "{{ rke2_token.content | b64decode | regex_replace('\n', '') }}" + +- name: Restore node-token file access + ansible.builtin.file: + path: /var/lib/rancher/rke2/server + mode: "{{ token_stats.stat.mode }}" + when: inventory_hostname == groups["server"][0] + become: true + +- name: Ensure .kube directory exists in user's home + ansible.builtin.file: + path: "/home/{{ ansible_user }}/.kube" + state: directory + mode: '0755' + become: true + +- name: Copy config file to user home directory + ansible.builtin.copy: + src: /etc/rancher/rke2/rke2.yaml + dest: "/home/{{ ansible_user }}/.kube/config" + remote_src: true + owner: "{{ ansible_user }}" + mode: "u=rw,g=,o=" + when: inventory_hostname == groups["server"][0] + become: true + +- name: Replace IP address with VIP + ansible.builtin.replace: + path: /home/{{ ansible_user }}/.kube/config + regexp: '127.0.0.1' + replace: "{{ vip }}" + when: inventory_hostname == groups["server"][0] + become: true diff --git a/tasks/join-servers.yml b/tasks/join-servers.yml new file mode 100644 index 0000000..48ff2cc --- /dev/null +++ b/tasks/join-servers.yml @@ -0,0 +1,61 @@ +- name: Create systemd service file for RKE2 server + ansible.builtin.template: + src: templates/rke2-server.service.j2 + dest: /etc/systemd/system/rke2-server.service + owner: root + group: root + mode: '0644' + when: inventory_hostname in groups['server'] and inventory_hostname != groups['server'][0] + become: true + +- name: Create systemd service file for RKE2 agent + ansible.builtin.template: + src: templates/rke2-agent.service.j2 + dest: /etc/systemd/system/rke2-agent.service + owner: root + group: root + mode: '0644' + when: inventory_hostname in groups['server'] and inventory_hostname != groups['server'][0] + become: true + +- name: Wait for cluster API to be ready (can take 5-10 mins depending on internet/hardware) + ansible.builtin.command: + cmd: "kubectl get nodes" + register: kubectl_output + until: "'connection refused' not in kubectl_output.stderr" + retries: 120 + delay: 10 + changed_when: true + when: inventory_hostname == groups['server'][0] + +# Use kubectl to deploy yaml. Perhaps this can be added to the manifest folder initially +- name: Apply kube vip configuration file + ansible.builtin.command: + cmd: kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml apply -f https://kube-vip.io/manifests/rbac.yaml + changed_when: true + when: inventory_hostname == groups['servers'][0] + +# Apply the kube-vip configration. Perhaps this can be added to the manifest folder initially +- name: Apply kube vip configuration file + ansible.builtin.command: + cmd: kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml + changed_when: true + when: inventory_hostname == groups['servers'][0] + +# Check that additional servers are restarted +- name: Ensure additional RKE2 servers are enabled and running + ansible.builtin.systemd: + name: rke2-server + enabled: true + state: restarted + daemon_reload: true + when: inventory_hostname != groups['servers'][0] + +# enable additional servers +- name: Ensure RKE2 server is enabled and running + ansible.builtin.systemd: + name: rke2-server + enabled: true + state: restarted + daemon_reload: true + when: inventory_hostname != groups['servers'][0] \ No newline at end of file diff --git a/tasks/kube-vip.yml b/tasks/kube-vip.yml new file mode 100644 index 0000000..0e23f80 --- /dev/null +++ b/tasks/kube-vip.yml @@ -0,0 +1,27 @@ +- name: Create directory for Kube VIP Manifest + ansible.builtin.file: + path: "/var/lib/rancher/rke2/server/manifests" + state: directory + mode: "0755" + when: inventory_hostname in groups['server'] + become: true + +- name: Deploy Kube VIP Configuration + ansible.builtin.template: + src: templates/kube-vip.yaml.j2 + dest: /var/lib/rancher/rke2/server/manifests/kube-vip.yaml + owner: root + group: root + mode: "0644" + when: inventory_hostname == groups['server'][0] + become: true + +- name: Deploy Kube VIP Configuration + ansible.builtin.template: + src: templates/kube-vip-controller.yaml.j2 + dest: /var/lib/rancher/rke2/server/manifests/kube-vip.yaml + owner: root + group: root + mode: "0644" + when: inventory_hostname == groups['server'][0] + become: true diff --git a/tasks/main.yml b/tasks/main.yml new file mode 100644 index 0000000..644a546 --- /dev/null +++ b/tasks/main.yml @@ -0,0 +1,33 @@ +--- +- name: Preparation of the nodes + ansible.builtin.include_tasks: prepare-nodes.yml + tags: prepare-nodes + +- name: Creation of the kube-vip manifest for HA and prepare for loadbalancing + ansible.builtin.include_tasks: kube-vip.yml + tags: kube-vip + +- name: Bootstrap the RKE cluster on the first node + ansible.builtin.include_tasks: bootstrap-rke.yml + tags: bootstrap-rke + +# # Adds additional servers using the token from the previous task +# - name: Add additional RKE2 Servers +# hosts: servers +# gather_facts: true +# roles: +# - add-server + +# # Adds agents to the cluster +# - name: Add additional RKE2 Agents +# hosts: agents +# gather_facts: true +# roles: +# - add-agent + +# # Finish kube-vip, add metallb +# - name: Apply manifests after cluster is created +# hosts: servers +# gather_facts: true +# roles: +# - apply-manifests diff --git a/tasks/prepare-nodes.yml b/tasks/prepare-nodes.yml new file mode 100644 index 0000000..5b360ca --- /dev/null +++ b/tasks/prepare-nodes.yml @@ -0,0 +1,36 @@ +- name: Enable IPv4 forwarding + ansible.posix.sysctl: + name: net.ipv4.ip_forward + value: "1" + state: present + reload: true + become: true + +- name: Enable IPv6 forwarding + ansible.posix.sysctl: + name: net.ipv6.conf.all.forwarding + value: "1" + state: present + reload: true + become: true + +- name: Create directory for RKE2 binary + ansible.builtin.file: + path: "{{ rke2_install_dir }}" + state: directory + mode: '0755' + become: true + +- name: Download RKE2 binary + ansible.builtin.get_url: + url: "{{ rke2_binary_url }}" + dest: "{{ rke2_install_dir }}/rke2" + mode: '0755' + become: true + +- name: Set executable permissions on the RKE2 binary + ansible.builtin.file: + path: "{{ rke2_install_dir }}/rke2" + mode: '0755' + state: file + become: true diff --git a/templates/kube-vip-controller.yaml.j2 b/templates/kube-vip-controller.yaml.j2 new file mode 100644 index 0000000..cebf310 --- /dev/null +++ b/templates/kube-vip-controller.yaml.j2 @@ -0,0 +1,88 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip-cloud-controller + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-cloud-controller-role +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "create", "update", "list", "put"] + - apiGroups: [""] + resources: ["configmaps", "endpoints","events","services/status", "leases"] + verbs: ["*"] + - apiGroups: [""] + resources: ["nodes", "services"] + verbs: ["list","get","watch","update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-cloud-controller-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-cloud-controller-role +subjects: +- kind: ServiceAccount + name: kube-vip-cloud-controller + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-vip-cloud-provider + namespace: kube-system +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: kube-vip + component: kube-vip-cloud-provider + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app: kube-vip + component: kube-vip-cloud-provider + spec: + containers: + - command: + - /kube-vip-cloud-provider + - --leader-elect-resource-name=kube-vip-cloud-controller + image: ghcr.io/kube-vip/kube-vip-cloud-provider:v0.0.10 + name: kube-vip-cloud-provider + imagePullPolicy: Always + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 + serviceAccountName: kube-vip-cloud-controller + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 10 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - weight: 10 + preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists \ No newline at end of file diff --git a/templates/kube-vip.yaml.j2 b/templates/kube-vip.yaml.j2 new file mode 100644 index 0000000..84fca51 --- /dev/null +++ b/templates/kube-vip.yaml.j2 @@ -0,0 +1,130 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: {{ kube_vip_version }} + name: kube-vip-ds + namespace: kube-system +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip-ds + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: {{ kube_vip_version }} + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: {{ vip_interface }} + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "false" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: {{ vip }} + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:{{ kube_vip_version }} + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["services", "endpoints"] + verbs: ["list","get","watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["list","get","watch", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["list","get","watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system \ No newline at end of file diff --git a/templates/rke2-agent.service.j2 b/templates/rke2-agent.service.j2 new file mode 100644 index 0000000..b9d46b5 --- /dev/null +++ b/templates/rke2-agent.service.j2 @@ -0,0 +1,13 @@ +# rke2-agent.service.j2 +[Unit] +Description=RKE2 Agent +After=network.target + +[Service] +ExecStart={{ rke2_install_dir }}/rke2 agent +KillMode=process +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/templates/rke2-server-config.j2 b/templates/rke2-server-config.j2 new file mode 100644 index 0000000..6b0cb06 --- /dev/null +++ b/templates/rke2-server-config.j2 @@ -0,0 +1,14 @@ +write-kubeconfig-mode: "0644" +{% if inventory_hostname != groups['server'][0] %} +token: {{ hostvars[groups['server'][0]]['token'] }} +server: https://{{ vip }}:9345 +{% endif %} +tls-san: + - {{ vip }} + {% for host in groups['server'] -%} + - {{ hostvars[host]['ansible_host'] }} + {%- endfor -%} +node-label: + - server=true +# disable: +# - rke2-ingress-nginx \ No newline at end of file diff --git a/templates/rke2-server.service.j2 b/templates/rke2-server.service.j2 new file mode 100644 index 0000000..bc9e0a4 --- /dev/null +++ b/templates/rke2-server.service.j2 @@ -0,0 +1,13 @@ +# rke2-server.service.j2 +[Unit] +Description=RKE2 server +After=network.target + +[Service] +ExecStart={{ rke2_install_dir }}/rke2 server +KillMode=process +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/tests/ansible.cfg b/tests/ansible.cfg new file mode 100644 index 0000000..6cc1c0b --- /dev/null +++ b/tests/ansible.cfg @@ -0,0 +1,4 @@ +[defaults] +inventory = inventory.yml +host_key_checking=False +roles_path=../../. \ No newline at end of file diff --git a/tests/inventory.yml b/tests/inventory.yml new file mode 100644 index 0000000..4cdacb2 --- /dev/null +++ b/tests/inventory.yml @@ -0,0 +1,19 @@ +--- +rke2-cluster: + vars: + ansible_user: user + children: + server: + hosts: + lx-rke2-ctl01: + ansible_host: 192.168.178.21 + lx-rke2-ctl02: + ansible_host: 192.168.178.22 + lx-rke2-ctl03: + ansible_host: 192.168.178.23 + agent: + hosts: + lx-rke2-wrk01: + ansible_host: 192.168.178.24 + lx-rke2-wrk02: + ansible_host: 192.168.178.2 diff --git a/tests/test.yml b/tests/test.yml new file mode 100644 index 0000000..d5e509d --- /dev/null +++ b/tests/test.yml @@ -0,0 +1,10 @@ +--- +- name: Install RKE2 HA cluster + hosts: rke2-cluster + remote_user: root + roles: + - rke2-installation + vars: + vip: "192.168.178.20" + rke2_version: "v1.31.2+rke2r1" + arch: "amd64"