From 9b62fdd7ac90c74bfab82cbe1bc413092e02e3e7 Mon Sep 17 00:00:00 2001 From: Nikita Kozlovskiy Date: Fri, 31 May 2024 13:26:31 +0200 Subject: [PATCH] move ydb-tech-ci cloud deploy manifests --- .../ansible/ydb-ci-cloud/ansible.cfg | 16 ++++ .../ansible/ydb-ci-cloud/bazel-remote.yaml | 6 ++ .../ydb-ci-cloud/host_vars/cachesrv.yaml | 12 +++ .../ansible/ydb-ci-cloud/hosts.yaml | 5 ++ .../roles/bazel-remote/defaults/main.yaml | 9 +++ .../roles/bazel-remote/handlers/main.yaml | 7 ++ .../roles/bazel-remote/tasks/htpasswd.yaml | 28 +++++++ .../roles/bazel-remote/tasks/main.yaml | 55 +++++++++++++ .../templates/bazel-remote.service.j2 | 26 ++++++ .../templates/bazel-remote.yaml.j2 | 2 + .../terraform/ydb-ci-cloud/.gitignore | 2 + .../ydb-ci-cloud/.terraform.lock.hcl | 17 ++++ .../terraform/ydb-ci-cloud/README.md | 10 +++ .../terraform/ydb-ci-cloud/ansible-vault.tf | 19 +++++ .../terraform/ydb-ci-cloud/cachesrv.tf | 66 +++++++++++++++ .../ydb-ci-cloud/get-backend-configuration.sh | 14 ++++ .../ydb-ci-cloud/gh-runner/clickhouse.tf | 66 +++++++++++++++ .../ydb-ci-cloud/gh-runner/controller.tf | 81 +++++++++++++++++++ .../ydb-ci-cloud/gh-runner/logging.tf | 5 ++ .../ydb-ci-cloud/gh-runner/outputs.tf | 3 + .../ydb-ci-cloud/gh-runner/providers.tf | 15 ++++ .../ydb-ci-cloud/gh-runner/secrets.tf | 20 +++++ .../ydb-ci-cloud/gh-runner/variables.tf | 56 +++++++++++++ .../ydb-ci-cloud/gh-runner/webhook.tf | 67 +++++++++++++++ .../terraform/ydb-ci-cloud/github-sa.tf | 28 +++++++ .../ydb-ci-cloud/terraform/ydb-ci-cloud/gw.tf | 37 +++++++++ .../ydb-ci-cloud/instance-metadata.tf | 16 ++++ .../terraform/ydb-ci-cloud/main.tf | 16 ++++ .../terraform/ydb-ci-cloud/net.tf | 31 +++++++ .../terraform/ydb-ci-cloud/outputs.tf | 17 ++++ .../terraform/ydb-ci-cloud/providers.tf | 15 ++++ .../terraform/ydb-ci-cloud/registry.tf | 4 + .../ydb-ci-cloud/s3-ydb-gh-canondata.tf | 51 ++++++++++++ .../terraform/ydb-ci-cloud/s3-ydb-gh-logs.tf | 55 +++++++++++++ .../terraform/ydb-ci-cloud/secrets.tf | 0 .../terraform/ydb-ci-cloud/variables.tf | 70 ++++++++++++++++ 36 files changed, 947 insertions(+) create mode 100644 ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/ansible.cfg create mode 100644 ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/bazel-remote.yaml create mode 100644 ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/host_vars/cachesrv.yaml create mode 100644 ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/hosts.yaml create mode 100644 ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/defaults/main.yaml create mode 100644 ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/handlers/main.yaml create mode 100644 ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/tasks/htpasswd.yaml create mode 100644 ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/tasks/main.yaml create mode 100644 ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/templates/bazel-remote.service.j2 create mode 100644 ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/templates/bazel-remote.yaml.j2 create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/.gitignore create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/.terraform.lock.hcl create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/README.md create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/ansible-vault.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/cachesrv.tf create mode 100755 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/get-backend-configuration.sh create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/clickhouse.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/controller.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/logging.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/outputs.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/providers.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/secrets.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/variables.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/webhook.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/github-sa.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gw.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/instance-metadata.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/main.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/net.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/outputs.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/providers.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/registry.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/s3-ydb-gh-canondata.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/s3-ydb-gh-logs.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/secrets.tf create mode 100644 ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/variables.tf diff --git a/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/ansible.cfg b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/ansible.cfg new file mode 100644 index 000000000000..0929045ad8c4 --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/ansible.cfg @@ -0,0 +1,16 @@ +[defaults] +forks = 10 +inventory = hosts.yaml +retry_files_enabled = False +interpreter_python=/usr/bin/python3 +callbacks_enabled = ansible.posix.profile_tasks +stdout_callback = yaml +stderr_callback = yaml +check_mode_markers = true +show_per_host_start = false +show_custom_stats = true + +roles_path = ./roles: + +[ssh_connection] +pipelining = true diff --git a/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/bazel-remote.yaml b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/bazel-remote.yaml new file mode 100644 index 000000000000..b0ce5ab31643 --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/bazel-remote.yaml @@ -0,0 +1,6 @@ +--- +- name: install bazel-remote + hosts: bazel_remote_servers + become: true + roles: + - bazel-remote diff --git a/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/host_vars/cachesrv.yaml b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/host_vars/cachesrv.yaml new file mode 100644 index 000000000000..20f1d6661a5a --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/host_vars/cachesrv.yaml @@ -0,0 +1,12 @@ +bazel_remote_htpasswd_lockbox_secret_id: e6qe20m48alkec2btn5v +bazel_remote_instances: + - name: ccache + config: + dir: /mnt/ccache/cache/ + max_size: 175 + http_address: 0.0.0.0:8080 + - name: ya-cache + config: + dir: /mnt/ya-cache/cache/ + max_size: 4000 + http_address: 0.0.0.0:8081 diff --git a/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/hosts.yaml b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/hosts.yaml new file mode 100644 index 000000000000..f3756f2a3530 --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/hosts.yaml @@ -0,0 +1,5 @@ +bazel_remote_servers: + hosts: + cachesrv: + ansible_host: 158.160.147.211 + diff --git a/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/defaults/main.yaml b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/defaults/main.yaml new file mode 100644 index 000000000000..4a17794589fd --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/defaults/main.yaml @@ -0,0 +1,9 @@ +bazel_remote_version: 2.4.3 +bazel_remote_config_default: + disable_http_ac_validation: true + allow_unauthenticated_reads: true + htpasswd_file: /home/bazel-remote/htpasswd + grpc_address: none + +bazel_remote_instances: [] +bazel_remote_htpasswd_lockbox_secret_id: ~ \ No newline at end of file diff --git a/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/handlers/main.yaml b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/handlers/main.yaml new file mode 100644 index 000000000000..26e153ae696b --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/handlers/main.yaml @@ -0,0 +1,7 @@ +- name: restart-bazel-remote + ansible.builtin.systemd_service: + name: "bazel-remote@{{ item.name }}" + enabled: true + state: restarted + loop: "{{ bazel_remote_instances }}" + \ No newline at end of file diff --git a/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/tasks/htpasswd.yaml b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/tasks/htpasswd.yaml new file mode 100644 index 000000000000..2f3bb08f4529 --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/tasks/htpasswd.yaml @@ -0,0 +1,28 @@ +- name: ensure YC_TOKEN variable exists + ansible.builtin.assert: + that: + - lookup('env', 'YC_TOKEN') != '' + msg: | + Please set YC_TOKEN environment variable, example: export YC_TOKEN=$(yc --profile ydbtech iam create-token) + +- name: get htpasswd content + delegate_to: 127.0.0.1 + become: false + ansible.builtin.uri: + url: "https://payload.lockbox.api.cloud.yandex.net/lockbox/v1/secrets/{{ bazel_remote_htpasswd_lockbox_secret_id }}/payload" + headers: + Authorization: "Bearer {{ lookup('env', 'YC_TOKEN') }}" + register: htpasswd + check_mode: no + +- name: extract htpasswd content + ansible.builtin.set_fact: + htpasswd_content: "{{ (htpasswd.json.entries | items2dict('key', 'textValue')).htpasswd }}" + +- name: create htpasswd + ansible.builtin.copy: + dest: /home/bazel-remote/htpasswd + content: "{{ htpasswd_content }}" + mode: 0600 + owner: bazel-remote + group: bazel-remote diff --git a/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/tasks/main.yaml b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/tasks/main.yaml new file mode 100644 index 000000000000..997b9b2a9a37 --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/tasks/main.yaml @@ -0,0 +1,55 @@ +- name: download bazel-remote + ansible.builtin.get_url: + url: "https://github.com/buchgr/bazel-remote/releases/download/v{{ bazel_remote_version }}/bazel-remote-{{ bazel_remote_version }}-linux-x86_64" + dest: /usr/local/bin/bazel-remote + mode: '0755' + +- name: add bazel-remote group + ansible.builtin.group: + name: bazel-remote + system: true + +- name: add bazel-remote user + ansible.builtin.user: + name: bazel-remote + group: bazel-remote + system: true + +- name: create folders + ansible.builtin.file: + path: "{{ item.config.dir }}" + state: directory + mode: 0755 + owner: bazel-remote + group: bazel-remote + loop: "{{ bazel_remote_instances }}" + +- ansible.builtin.include_tasks: htpasswd.yaml + +- name: configure instances + ansible.builtin.template: + src: bazel-remote.yaml.j2 + dest: "/usr/local/etc/bazel-remote-{{ item.name }}.yaml" + notify: + - restart-bazel-remote + loop: "{{ bazel_remote_instances }}" + +- name: configure systemd unit + ansible.builtin.template: + src: bazel-remote.service.j2 + dest: "/etc/systemd/system/bazel-remote@.service" + register: systemd_unit + notify: + - restart-bazel-remote + +- name: reload systemd daemon + ansible.builtin.systemd_service: + daemon-reload: true + when: systemd_unit.changed + +- name: enable systemd units + ansible.builtin.systemd_service: + name: "bazel-remote@{{ item.name }}" + enabled: true + loop: "{{ bazel_remote_instances }}" + \ No newline at end of file diff --git a/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/templates/bazel-remote.service.j2 b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/templates/bazel-remote.service.j2 new file mode 100644 index 000000000000..27436af61ac7 --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/templates/bazel-remote.service.j2 @@ -0,0 +1,26 @@ +[Unit] +Description=bazel-remote cache (%i instance) + +[Service] +# Assuming you have created a bazel-remote user and group, that can write +# to the cache directory specified in ExecStart below: +User=bazel-remote +Group=bazel-remote + +# We need to have a lot of files open at once. +LimitNOFILE=1000000 + +# Try to avoid "runtime: failed to create new OS thread (have 2458 already; errno=11)" +# errors. You can check if this worked by running "systemctl status bazel-remote" +# and see if there's a "Tasks: 18 (limit: 2457)" line (hopefully not, after adding this). +LimitNPROC=infinity +TasksMax=infinity + +Restart=on-failure + +Environment=GODEBUG=gctrace=1 + +ExecStart=/usr/local/bin/bazel-remote --config_file /usr/local/etc/bazel-remote-%i.yaml + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/templates/bazel-remote.yaml.j2 b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/templates/bazel-remote.yaml.j2 new file mode 100644 index 000000000000..2a70518487fb --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/templates/bazel-remote.yaml.j2 @@ -0,0 +1,2 @@ +# Bazel remote config for {{ item.name }}, generated using ansible. Please don't modify by hand. +{{ (bazel_remote_config_default | ansible.builtin.combine(item.config)) | to_nice_yaml }} \ No newline at end of file diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/.gitignore b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/.gitignore new file mode 100644 index 000000000000..8b9d408f4099 --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/.gitignore @@ -0,0 +1,2 @@ +.terraform +backend-configuration.tf diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/.terraform.lock.hcl b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/.terraform.lock.hcl new file mode 100644 index 000000000000..588da85c744b --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/.terraform.lock.hcl @@ -0,0 +1,17 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/random" { + version = "3.6.2" + hashes = [ + "h1:VavG5unYCa3SYISMKF9pzc3718M0bhPlcbUZZGl7wuo=", + ] +} + +provider "registry.terraform.io/yandex-cloud/yandex" { + version = "0.119.0" + constraints = ">= 0.71.0" + hashes = [ + "h1:su5K+VQhF6bgfCNux6wjx3yLi1ZqK8gwgWXXxviWZnc=", + ] +} diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/README.md b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/README.md new file mode 100644 index 000000000000..73255c80fbb4 --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/README.md @@ -0,0 +1,10 @@ +``` +export YC_TOKEN=$(yc --profile=ydbtech iam create-token) +./get-backend-configuration.sh +terraform init +``` + + +terraform yandex provider limitations: +1. Make the serverless container public +2. Make a revision with 1 Always on prepared container \ No newline at end of file diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/ansible-vault.tf b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/ansible-vault.tf new file mode 100644 index 000000000000..28ded3e7022e --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/ansible-vault.tf @@ -0,0 +1,19 @@ +resource "random_password" "ansible-vault" { + length = 16 + special = false +} + +resource "yandex_lockbox_secret" "ansible-vault" { + name = "ansible-vault" + description = "ansible-vault key" +} + + +resource "yandex_lockbox_secret_version" "ansible-vault" { + secret_id = yandex_lockbox_secret.ansible-vault.id + entries { + key = "key" + text_value = random_password.ansible-vault.result + } + +} \ No newline at end of file diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/cachesrv.tf b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/cachesrv.tf new file mode 100644 index 000000000000..0582c9a877a3 --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/cachesrv.tf @@ -0,0 +1,66 @@ +resource "yandex_compute_disk" "cachesrv-ccache" { + name = "cachesrv-ccache" + zone = var.yc_zone + type = "network-ssd-nonreplicated" + size = 2 * 93 +} + +resource "yandex_compute_disk" "cachesrv-ya" { + name = "cachesrv-ya" + zone = var.yc_zone + type = "network-ssd-nonreplicated" + size = 45 * 93 +} + +resource "yandex_vpc_address" "cachesrv" { + name = "cachesrv external ip" + deletion_protection = true + + external_ipv4_address { + zone_id = var.yc_zone + } +} + +resource "yandex_compute_instance" "cachesrv" { + name = "cachesrv" + platform_id = "standard-v3" + zone = var.yc_zone + + resources { + cores = 32 + memory = 96 + core_fraction = 100 + } + + boot_disk { + initialize_params { + type = "network-ssd" + size = 64 + image_id = var.cachesrv-image-id + } + } + secondary_disk { + disk_id = yandex_compute_disk.cachesrv-ccache.id + device_name = "ccache" + } + + secondary_disk { + disk_id = yandex_compute_disk.cachesrv-ya.id + device_name = "ya-cache" + } + + network_interface { + subnet_id = yandex_vpc_subnet.default[var.yc_zone].id + nat = true + nat_ip_address = yandex_vpc_address.cachesrv.external_ipv4_address[0].address + dns_record { + fqdn = "cachesrv.${var.dns_zone_fqdn}." + ptr = true + } + } + + metadata = { + serial-port-enable : "1" + user-data = local.instance-metadata + } +} diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/get-backend-configuration.sh b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/get-backend-configuration.sh new file mode 100755 index 000000000000..bf12ec9f92db --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/get-backend-configuration.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +#function create_secret { +# TF_VAR_cloud_id=b1ggceeul2pkher8vhb6 \ +# TF_VAR_folder_id=b1grf3mpoatgflnlavjd \ +# TF_VAR_instance=ydbtech \ +# TF_VAR_yc_endpoint="api.cloud.yandex.net:443" \ +# TF_VAR_yc_storage_endpoint="storage.yandexcloud.net:443" \ +# terraform apply +#} + +yc --profile ydbtech --endpoint api.cloud.yandex.net:443 \ + --folder-id b1grf3mpoatgflnlavjd --cloud-id b1ggceeul2pkher8vhb6 \ + lockbox payload get --key config e6q75n7s571uk3f6oemc > backend-configuration.tf diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/clickhouse.tf b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/clickhouse.tf new file mode 100644 index 000000000000..22a2d96d4a2f --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/clickhouse.tf @@ -0,0 +1,66 @@ +resource "random_password" "ch-password" { + length = 16 + special = false +} + +resource "yandex_mdb_clickhouse_cluster" "jobs" { + name = "gh-jobs" + environment = "PRODUCTION" + network_id = var.network_id + + + + clickhouse { + resources { + resource_preset_id = "s3-c2-m8" + disk_type_id = "network-ssd" + disk_size = 128 + } + } + + access { + web_sql = true + data_lens = true + } + + database { + name = var.ch-dbname + } + + host { + type = "CLICKHOUSE" + zone = var.yc_zone + subnet_id = var.subnet_id + } + + + user { + name = var.ch-username + # FIXME: password leak via terraform state + password = random_password.ch-password.result + permission { + database_name = var.ch-dbname + } + } +} + +resource "yandex_lockbox_secret_version" "clickhouse" { + secret_id = yandex_lockbox_secret.secrets.id + + entries { + key = "ch_fqdns" + text_value = join(",", yandex_mdb_clickhouse_cluster.jobs.host[*].fqdn) + } + entries { + key = "ch_database" + text_value = var.ch-dbname + } + entries { + key = "ch_username" + text_value = var.ch-username + } + entries { + key = "ch_password" + text_value = random_password.ch-password.result + } +} diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/controller.tf b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/controller.tf new file mode 100644 index 000000000000..d27fdcb0a154 --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/controller.tf @@ -0,0 +1,81 @@ +resource "yandex_iam_service_account" "runner-controller" { + name = "gh-runner-runner-sa" +} + +resource "yandex_resourcemanager_folder_iam_binding" "compute-sa-binding" { + folder_id = var.folder_id + members = [ + "serviceAccount:${yandex_iam_service_account.runner-controller.id}" + ] + role = "compute.editor" +} + +resource "yandex_lockbox_secret_iam_binding" "controller-gh-lockbox-binding" { + members = [ + "serviceAccount:${yandex_iam_service_account.runner-controller.id}" + ] + role = "lockbox.payloadViewer" + secret_id = yandex_lockbox_secret.github-secrets.id +} + +resource "yandex_lockbox_secret_iam_binding" "controller-lockbox-secrets-binding" { + members = [ + "serviceAccount:${yandex_iam_service_account.runner-controller.id}" + ] + secret_id = yandex_lockbox_secret.secrets.id + role = "lockbox.payloadViewer" +} + + +resource "yandex_container_registry_iam_binding" "controller-registry-iam-binding" { + registry_id = var.registry_id + role = "container-registry.images.puller" + members = [ + "serviceAccount:${yandex_iam_service_account.runner-controller.id}" + ] +} + +resource "yandex_resourcemanager_folder_iam_binding" "controller-log-iam-folder-binding" { + folder_id = var.folder_id + role = "logging.writer" + members = [ + "serviceAccount:${yandex_iam_service_account.runner-controller.id}" + ] + +} + + +resource "yandex_compute_instance" "controller" { + name = "gh-runners-controller" + platform_id = "standard-v3" + zone = var.yc_zone + + resources { + cores = 2 + memory = 2 + core_fraction = 100 + } + + boot_disk { + initialize_params { + image_id = var.controller-image-id + } + } + + network_interface { + subnet_id = var.subnet_id + dns_record { + fqdn = "gh-runners-controller.${var.dns_zone}." + ptr = true + } + } + + service_account_id = yandex_iam_service_account.runner-controller.id + + metadata = { + serial-port-enable : "1" + user-data = var.instance-metadata + } +} + + diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/logging.tf b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/logging.tf new file mode 100644 index 000000000000..7b57c862e619 --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/logging.tf @@ -0,0 +1,5 @@ +resource "yandex_logging_group" "controller-logs" { + folder_id = var.folder_id + name = "gh-runners-controller-logs" + retention_period = "168h" +} diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/outputs.tf b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/outputs.tf new file mode 100644 index 000000000000..d5df87285cc8 --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/outputs.tf @@ -0,0 +1,3 @@ +output "controller-internal-ip" { + value = yandex_compute_instance.controller.network_interface[0].ip_address +} diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/providers.tf b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/providers.tf new file mode 100644 index 000000000000..4b360be694df --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/providers.tf @@ -0,0 +1,15 @@ +terraform { + required_providers { + yandex = { + source = "yandex-cloud/yandex" + version = ">= 0.71.0" + } + } +} + +provider "yandex" { + endpoint = var.yc_endpoint + cloud_id = var.cloud_id + folder_id = var.folder_id + zone = var.yc_zone +} diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/secrets.tf b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/secrets.tf new file mode 100644 index 000000000000..4fbba2a48f4a --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/secrets.tf @@ -0,0 +1,20 @@ +resource "yandex_lockbox_secret" "secrets" { + name = "gh-runner-secrets" + description = "Secrets for Managed services, like Clickhouse" +} + +resource "yandex_lockbox_secret" "github-secrets" { + name = "gh-runner-secrets-github-secrets" + description = "Lockbox for GitHub Tokens and keys. All versions are updated by hands." +} + +data "yandex_lockbox_secret" "github-secrets-version" { + secret_id = yandex_lockbox_secret.github-secrets.id + + lifecycle { + postcondition { + condition = contains(self.current_version[0].payload_entry_keys, "GH_WEBHOOK_SECRET") + error_message = "You must add GH_WEBHOOK_SECRET to the gh-runner-secrets-github-secrets lockbox" + } + } +} diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/variables.tf b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/variables.tf new file mode 100644 index 000000000000..90ee58e22d29 --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/variables.tf @@ -0,0 +1,56 @@ +variable "yc_endpoint" { + type = string +} + +variable "cloud_id" { + type = string +} + +variable "folder_id" { + type = string +} + +variable "yc_zone" { + type = string +} + +variable "registry_id" { + type = string +} + +variable "network_id" { + type = string +} + +variable "subnet_id" { + type = string +} + +variable "dns_zone" { + type = string +} + +variable "instance-metadata" { + type = string +} + +variable "ch-dbname" { + type = string + default = "gh" +} + +variable "ch-username" { + type = string + default = "gh" +} + +variable "controller-image-id" { + type = string + # ubuntu-2204-lts-oslogin + default = "fd8bu9gsckcm2351kqaq" +} + +variable "webhook_container_image" { + type = string +} + diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/webhook.tf b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/webhook.tf new file mode 100644 index 000000000000..4377e25316cd --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/webhook.tf @@ -0,0 +1,67 @@ +resource "yandex_iam_service_account" "gh-runner-webhook-sa" { + name = "gh-runner-webhook-sa" +} + +resource "yandex_container_registry_iam_binding" "gh-runner-iam-binding" { + registry_id = var.registry_id + role = "container-registry.images.puller" + members = [ + "serviceAccount:${yandex_iam_service_account.gh-runner-webhook-sa.id}" + ] +} + +resource "yandex_lockbox_secret_iam_binding" "gh-runner-iam-binding" { + secret_id = yandex_lockbox_secret.secrets.id + for_each = toset(["lockbox.payloadViewer", "lockbox.viewer"]) + role = each.value + members = [ + "serviceAccount:${yandex_iam_service_account.gh-runner-webhook-sa.id}" + ] +} + +resource "yandex_lockbox_secret_iam_binding" "gh-runner-iam-github-binding" { + secret_id = yandex_lockbox_secret.github-secrets.id + for_each = toset(["lockbox.payloadViewer", "lockbox.viewer"]) + role = each.value + members = [ + "serviceAccount:${yandex_iam_service_account.gh-runner-webhook-sa.id}" + ] +} + + +resource "yandex_serverless_container" "gh-runner-webhook" { + name = "gh-runner-github-webhook" + description = "PUBLIC Container for receiving GitHub (https://github.com/ydb-platform/ydb) webhooks about Jobs" + cores = 1 + core_fraction = 100 + memory = 256 + concurrency = 4 + + execution_timeout = "90s" + + service_account_id = yandex_iam_service_account.gh-runner-webhook-sa.id + + dynamic "secrets" { + for_each = yandex_lockbox_secret_version.clickhouse.entries + content { + environment_variable = upper(secrets.value.key) + id = yandex_lockbox_secret.secrets.id + version_id = yandex_lockbox_secret_version.clickhouse.id + key = secrets.value.key + } + } + secrets { + environment_variable = "GH_WEBHOOK_SECRET" + id = yandex_lockbox_secret.github-secrets.id + key = "GH_WEBHOOK_SECRET" + version_id = data.yandex_lockbox_secret.github-secrets-version.current_version.0.id + } + + connectivity { + network_id = var.network_id + } + + image { + url = var.webhook_container_image + } +} diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/github-sa.tf b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/github-sa.tf new file mode 100644 index 000000000000..def67b47bfda --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/github-sa.tf @@ -0,0 +1,28 @@ +resource "yandex_iam_service_account" "s3-ydb-gh" { + name = "s3-ydb-gh" + description = "ydb github s3 SA" +} + +resource "yandex_iam_service_account_static_access_key" "s3-ydb-gh" { + service_account_id = yandex_iam_service_account.s3-ydb-gh.id + description = "static access key for github" +} + + +resource "yandex_lockbox_secret" "s3-ydb-gh-sa" { + name = "s3-ydb-gh-sa" + description = "Keys for s3-ydb-gh SA" +} + +resource "yandex_lockbox_secret_version" "s3-ydb-gh-sa" { + secret_id = yandex_lockbox_secret.s3-ydb-gh-sa.id + + entries { + key = "access_key" + text_value = yandex_iam_service_account_static_access_key.s3-ydb-gh.access_key + } + entries { + key = "secret_key" + text_value = yandex_iam_service_account_static_access_key.s3-ydb-gh.secret_key + } +} \ No newline at end of file diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gw.tf b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gw.tf new file mode 100644 index 000000000000..5c5e949a5846 --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gw.tf @@ -0,0 +1,37 @@ +resource "yandex_vpc_address" "gw" { + name = "gw" + deletion_protection = true + + external_ipv4_address { + zone_id = var.yc_zone + } +} + +resource "yandex_compute_instance" "gw" { + name = "gh-runners-gw" + platform_id = "standard-v3" + zone = var.yc_zone + + resources { + cores = 2 + memory = 1 + core_fraction = 20 + } + + boot_disk { + initialize_params { + image_id = var.gw-image + } + } + + network_interface { + subnet_id = yandex_vpc_subnet.default[var.yc_zone].id + nat = true + nat_ip_address = yandex_vpc_address.gw.external_ipv4_address[0].address + } + + metadata = { + serial-port-enable : "1" + user-data = local.instance-metadata + } +} diff --git a/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/instance-metadata.tf b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/instance-metadata.tf new file mode 100644 index 000000000000..a4fae6a66876 --- /dev/null +++ b/ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/instance-metadata.tf @@ -0,0 +1,16 @@ +data "yandex_lockbox_secret_version" "instance-metadata-ssh-keys" { + secret_id = var.ssh-keys-lockbox-secret-id + version_id = var.ssh-keys-lockbox-version-id +} + +locals { + lockbox-instance-metadata-contents = { + for k, v in data.yandex_lockbox_secret_version.instance-metadata-ssh-keys.entries : v.key => v.text_value + } + instance-metadata = <