move ydb-tech-ci cloud deploy manifests into ydb/ci folder (#5077)

ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/ansible.cfg

@@ -0,0 +1,16 @@
+[defaults]
+forks = 10
+inventory = hosts.yaml
+retry_files_enabled = False
+interpreter_python=/usr/bin/python3
+callbacks_enabled = ansible.posix.profile_tasks
+stdout_callback = yaml
+stderr_callback = yaml
+check_mode_markers = true
+show_per_host_start = false
+show_custom_stats = true
+
+roles_path = ./roles:
+
+[ssh_connection]
+pipelining = true

ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/bazel-remote.yaml

@@ -0,0 +1,6 @@
+---
+- name: install bazel-remote
+  hosts: bazel_remote_servers
+  become: true
+  roles:
+    - bazel-remote

ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/host_vars/cachesrv.yaml

@@ -0,0 +1,12 @@
+bazel_remote_htpasswd_lockbox_secret_id: e6qe20m48alkec2btn5v
+bazel_remote_instances:
+  - name: ccache
+    config:
+      dir: /mnt/ccache/cache/
+      max_size: 175
+      http_address: 0.0.0.0:8080
+  - name: ya-cache
+    config:
+      dir: /mnt/ya-cache/cache/
+      max_size: 4000
+      http_address: 0.0.0.0:8081

ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/hosts.yaml

@@ -0,0 +1,5 @@
+bazel_remote_servers:
+  hosts:
+    cachesrv:
+      ansible_host: 158.160.147.211
+

ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/defaults/main.yaml

@@ -0,0 +1,9 @@
+bazel_remote_version: 2.4.3
+bazel_remote_config_default:
+  disable_http_ac_validation: true
+  allow_unauthenticated_reads: true
+  htpasswd_file: /home/bazel-remote/htpasswd
+  grpc_address: none
+
+bazel_remote_instances: []
+bazel_remote_htpasswd_lockbox_secret_id: ~

ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/handlers/main.yaml

@@ -0,0 +1,7 @@
+- name: restart-bazel-remote
+  ansible.builtin.systemd_service:
+    name: "bazel-remote@{{ item.name }}"
+    enabled: true
+    state: restarted
+  loop: "{{ bazel_remote_instances }}"
+

ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/tasks/htpasswd.yaml

@@ -0,0 +1,28 @@
+- name: ensure YC_TOKEN variable exists
+  ansible.builtin.assert:
+    that:
+      - lookup('env', 'YC_TOKEN') != ''
+    msg: |
+      Please set YC_TOKEN environment variable, example: export YC_TOKEN=$(yc --profile ydbtech iam create-token)
+
+- name: get htpasswd content
+  delegate_to: 127.0.0.1
+  become: false
+  ansible.builtin.uri:
+    url: "https://payload.lockbox.api.cloud.yandex.net/lockbox/v1/secrets/{{ bazel_remote_htpasswd_lockbox_secret_id }}/payload"
+    headers:
+      Authorization: "Bearer {{ lookup('env', 'YC_TOKEN') }}"
+  register: htpasswd
+  check_mode: no
+
+- name: extract htpasswd content
+  ansible.builtin.set_fact:
+    htpasswd_content: "{{ (htpasswd.json.entries | items2dict('key', 'textValue')).htpasswd  }}"
+
+- name: create htpasswd
+  ansible.builtin.copy:
+    dest: /home/bazel-remote/htpasswd
+    content: "{{ htpasswd_content }}"
+    mode: 0600
+    owner: bazel-remote
+    group: bazel-remote

ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/tasks/main.yaml

@@ -0,0 +1,55 @@
+- name: download bazel-remote
+  ansible.builtin.get_url:
+    url: "https://github.com/buchgr/bazel-remote/releases/download/v{{ bazel_remote_version }}/bazel-remote-{{ bazel_remote_version }}-linux-x86_64"
+    dest: /usr/local/bin/bazel-remote
+    mode: '0755'
+
+- name: add bazel-remote group
+  ansible.builtin.group:
+    name: bazel-remote
+    system: true
+
+- name: add bazel-remote user
+  ansible.builtin.user:
+    name: bazel-remote
+    group: bazel-remote
+    system: true
+
+- name: create folders
+  ansible.builtin.file:
+    path: "{{ item.config.dir }}"
+    state: directory
+    mode: 0755
+    owner: bazel-remote
+    group: bazel-remote
+  loop: "{{ bazel_remote_instances }}"
+
+- ansible.builtin.include_tasks: htpasswd.yaml
+
+- name: configure instances
+  ansible.builtin.template:
+    src: bazel-remote.yaml.j2
+    dest: "/usr/local/etc/bazel-remote-{{ item.name }}.yaml"
+  notify:
+    - restart-bazel-remote
+  loop: "{{ bazel_remote_instances }}"
+
+- name: configure systemd unit
+  ansible.builtin.template:
+    src: bazel-remote.service.j2
+    dest: "/etc/systemd/system/bazel-remote@.service"
+  register: systemd_unit
+  notify:
+    - restart-bazel-remote
+
+- name: reload systemd daemon
+  ansible.builtin.systemd_service:
+    daemon-reload: true
+  when: systemd_unit.changed
+
+- name: enable systemd units
+  ansible.builtin.systemd_service:
+    name: "bazel-remote@{{ item.name }}"
+    enabled: true
+  loop: "{{ bazel_remote_instances }}"
+

ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/templates/bazel-remote.service.j2

@@ -0,0 +1,26 @@
+[Unit]
+Description=bazel-remote cache (%i instance)
+
+[Service]
+# Assuming you have created a bazel-remote user and group, that can write
+# to the cache directory specified in ExecStart below:
+User=bazel-remote
+Group=bazel-remote
+
+# We need to have a lot of files open at once.
+LimitNOFILE=1000000
+
+# Try to avoid "runtime: failed to create new OS thread (have 2458 already; errno=11)"
+# errors. You can check if this worked by running "systemctl status bazel-remote"
+# and see if there's a "Tasks: 18 (limit: 2457)" line (hopefully not, after adding this).
+LimitNPROC=infinity
+TasksMax=infinity
+
+Restart=on-failure
+
+Environment=GODEBUG=gctrace=1
+
+ExecStart=/usr/local/bin/bazel-remote --config_file /usr/local/etc/bazel-remote-%i.yaml
+
+[Install]
+WantedBy=multi-user.target

ydb/ci/ydb-ci-cloud/ansible/ydb-ci-cloud/roles/bazel-remote/templates/bazel-remote.yaml.j2

@@ -0,0 +1,2 @@
+# Bazel remote config for {{  item.name }}, generated using ansible. Please don't modify by hand.
+{{ (bazel_remote_config_default | ansible.builtin.combine(item.config)) | to_nice_yaml }}

ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/.gitignore

@@ -0,0 +1,2 @@
+.terraform
+backend-configuration.tf

ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/README.md

@@ -0,0 +1,10 @@
+```
+export YC_TOKEN=$(yc --profile=ydbtech iam create-token)
+./get-backend-configuration.sh
+terraform init
+```
+
+
+terraform yandex provider limitations:
+1. Make the serverless container public
+2. Make a revision with 1 Always on prepared container

ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/ansible-vault.tf

@@ -0,0 +1,19 @@
+resource "random_password" "ansible-vault" {
+  length  = 16
+  special = false
+}
+
+resource "yandex_lockbox_secret" "ansible-vault" {
+  name        = "ansible-vault"
+  description = "ansible-vault key"
+}
+
+
+resource "yandex_lockbox_secret_version" "ansible-vault" {
+  secret_id = yandex_lockbox_secret.ansible-vault.id
+  entries {
+    key        = "key"
+    text_value = random_password.ansible-vault.result
+  }
+
+}

ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/cachesrv.tf

@@ -0,0 +1,66 @@
+resource "yandex_compute_disk" "cachesrv-ccache" {
+  name = "cachesrv-ccache"
+  zone = var.yc_zone
+  type = "network-ssd-nonreplicated"
+  size = 2 * 93
+}
+
+resource "yandex_compute_disk" "cachesrv-ya" {
+  name = "cachesrv-ya"
+  zone = var.yc_zone
+  type = "network-ssd-nonreplicated"
+  size = 45 * 93
+}
+
+resource "yandex_vpc_address" "cachesrv" {
+  name                = "cachesrv external ip"
+  deletion_protection = true
+
+  external_ipv4_address {
+    zone_id = var.yc_zone
+  }
+}
+
+resource "yandex_compute_instance" "cachesrv" {
+  name        = "cachesrv"
+  platform_id = "standard-v3"
+  zone        = var.yc_zone
+
+  resources {
+    cores         = 32
+    memory        = 96
+    core_fraction = 100
+  }
+
+  boot_disk {
+    initialize_params {
+      type     = "network-ssd"
+      size     = 64
+      image_id = var.cachesrv-image-id
+    }
+  }
+  secondary_disk {
+    disk_id     = yandex_compute_disk.cachesrv-ccache.id
+    device_name = "ccache"
+  }
+
+  secondary_disk {
+    disk_id     = yandex_compute_disk.cachesrv-ya.id
+    device_name = "ya-cache"
+  }
+
+  network_interface {
+    subnet_id      = yandex_vpc_subnet.default[var.yc_zone].id
+    nat            = true
+    nat_ip_address = yandex_vpc_address.cachesrv.external_ipv4_address[0].address
+    dns_record {
+      fqdn = "cachesrv.${var.dns_zone_fqdn}."
+      ptr  = true
+    }
+  }
+
+  metadata = {
+    serial-port-enable : "1"
+    user-data = local.instance-metadata
+  }
+}

ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/get-backend-configuration.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+#function create_secret {
+#  TF_VAR_cloud_id=b1ggceeul2pkher8vhb6 \
+#  TF_VAR_folder_id=b1grf3mpoatgflnlavjd \
+#  TF_VAR_instance=ydbtech \
+#  TF_VAR_yc_endpoint="api.cloud.yandex.net:443" \
+#  TF_VAR_yc_storage_endpoint="storage.yandexcloud.net:443" \
+#  terraform apply
+#}
+
+yc --profile ydbtech --endpoint api.cloud.yandex.net:443 \
+  --folder-id b1grf3mpoatgflnlavjd --cloud-id b1ggceeul2pkher8vhb6 \
+    lockbox payload get --key config e6q75n7s571uk3f6oemc > backend-configuration.tf

ydb/ci/ydb-ci-cloud/terraform/ydb-ci-cloud/gh-runner/clickhouse.tf

@@ -0,0 +1,66 @@
+resource "random_password" "ch-password" {
+  length  = 16
+  special = false
+}
+
+resource "yandex_mdb_clickhouse_cluster" "jobs" {
+  name        = "gh-jobs"
+  environment = "PRODUCTION"
+  network_id  = var.network_id
+
+
+
+  clickhouse {
+    resources {
+      resource_preset_id = "s3-c2-m8"
+      disk_type_id       = "network-ssd"
+      disk_size          = 128
+    }
+  }
+
+  access {
+    web_sql   = true
+    data_lens = true
+  }
+
+  database {
+    name = var.ch-dbname
+  }
+
+  host {
+    type      = "CLICKHOUSE"
+    zone      = var.yc_zone
+    subnet_id = var.subnet_id
+  }
+
+
+  user {
+    name = var.ch-username
+    # FIXME: password leak via terraform state
+    password = random_password.ch-password.result
+    permission {
+      database_name = var.ch-dbname
+    }
+  }
+}
+
+resource "yandex_lockbox_secret_version" "clickhouse" {
+  secret_id = yandex_lockbox_secret.secrets.id
+
+  entries {
+    key        = "ch_fqdns"
+    text_value = join(",", yandex_mdb_clickhouse_cluster.jobs.host[*].fqdn)
+  }
+  entries {
+    key        = "ch_database"
+    text_value = var.ch-dbname
+  }
+  entries {
+    key        = "ch_username"
+    text_value = var.ch-username
+  }
+  entries {
+    key        = "ch_password"
+    text_value = random_password.ch-password.result
+  }
+}