diff --git a/.github/actions/build_ya/action.yml b/.github/actions/build_ya/action.yml index 2869db1eb766..96e35551da50 100644 --- a/.github/actions/build_ya/action.yml +++ b/.github/actions/build_ya/action.yml @@ -145,6 +145,15 @@ runs: echo "Build successful." | .github/scripts/tests/comment-pr.py --ok fi + - name: build_stats + shell: bash + continue-on-error: true + run: | + set -x + export build_preset="${{ inputs.build_preset }}" + python3 -m pip install ydb ydb[yc] + python3 .github/scripts/send_build_stats.py + - name: show free space if: always() shell: bash diff --git a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml new file mode 100644 index 000000000000..5cac1a347915 --- /dev/null +++ b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml @@ -0,0 +1,19 @@ +name: setup_ci_ydb_service_account_key_file_credentials +description: setup environment for access to ydb instance in cloud +inputs: + ci_ydb_service_account_key_file_credentials: + required: false + description: "token for access" +runs: + using: "composite" + steps: + - name: drop file and set environment variable + if: "${{ inputs.ci_ydb_service_account_key_file_credentials != '' }}" + shell: bash + run: | + set -eu + export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=/tmp/ydb_service_account.json + cat << EOF > $YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS + ${{ inputs.ci_ydb_service_account_key_file_credentials}} + EOF + echo "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" >> $GITHUB_ENV diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py new file mode 100755 index 000000000000..5b1958b10af6 --- /dev/null +++ b/.github/scripts/send_build_stats.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 + +import datetime +import os +import ydb +import uuid +import subprocess + + +YDBD_PATH = "ydb/apps/ydbd/ydbd" + +FROM_ENV_COLUMNS = [ + "github_head_ref", + "github_workflow", + "github_workflow_ref", + "github_sha", + "github_repository", + "github_event_name", + "github_ref_type", + "github_ref_name", + "github_ref", +] + +STRING_COLUMNS = FROM_ENV_COLUMNS + [ + "id", + "git_commit_message", + "binary_path", + "build_preset", +] + +DATETIME_COLUMNS = [ + "git_commit_time", +] + +UINT64_COLUMNS = [ + "size_bytes", + "size_stripped_bytes", +] + +ALL_COLUMNS = STRING_COLUMNS + DATETIME_COLUMNS + UINT64_COLUMNS + + +def sanitize_str(s): + # YDB SDK expects bytes for 'String' columns + if s is None: + s = "N\A" + return s.encode("utf-8") + + +def main(): + if "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" not in os.environ: + print("Env variable YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS is missing, skipping") + return 1 + + with ydb.Driver( + endpoint="grpcs://ydb.serverless.yandexcloud.net:2135", + database="/ru-central1/b1ggceeul2pkher8vhb6/etn6d1qbals0c29ho4lf", + credentials=ydb.credentials_from_env_variables() + ) as driver: + driver.wait(timeout=10, fail_fast=True) + session = ydb.retry_operation_sync( + lambda: driver.table_client.session().create() + ) + with session.transaction() as tx: + text_query_builder = [] + for type_ in STRING_COLUMNS: + text_query_builder.append("DECLARE ${} as String;".format(type_)) + for type_ in UINT64_COLUMNS: + text_query_builder.append("DECLARE ${} as Uint64;".format(type_)) + for type_ in DATETIME_COLUMNS: + text_query_builder.append("DECLARE ${} as Datetime;".format(type_)) + + text_query_builder.append( + """INSERT INTO binary_size +( + {} +) +VALUES +( + {} +); +""".format( + ", \n ".join(ALL_COLUMNS), + ", \n ".join(["$" + column for column in ALL_COLUMNS]), + ) + ) + + text_query = "\n".join(text_query_builder) + + prepared_query = session.prepare(text_query) + + binary_size_bytes = subprocess.check_output( + ["bash", "-c", "cat {} | wc -c".format(YDBD_PATH)] + ) + binary_size_stripped_bytes = subprocess.check_output( + ["bash", "-c", "./ya tool strip {} -o - | wc -c".format(YDBD_PATH)] + ) + + build_preset = os.environ.get("build_preset", None) + github_sha = os.environ.get("GITHUB_SHA", None) + + if github_sha is not None: + git_commit_time_bytes = subprocess.check_output( + ["git", "show", "--no-patch", "--format=%cI", github_sha] + ) + git_commit_message_bytes = subprocess.check_output( + ["git", "log", "--format=%s", "-n", "1", github_sha] + ) + git_commit_time = datetime.datetime.fromisoformat( + git_commit_time_bytes.decode("utf-8").strip() + ) + git_commit_message = git_commit_message_bytes.decode("utf-8").strip() + git_commit_time_unix = int(git_commit_time.timestamp()) + else: + git_commit_time = None + git_commit_message = None + git_commit_time_unix = 0 + + parameters = { + "$id": sanitize_str(str(uuid.uuid4())), + "$build_preset": sanitize_str(build_preset), + "$binary_path": sanitize_str(YDBD_PATH), + "$size_stripped_bytes": int(binary_size_stripped_bytes.decode("utf-8")), + "$size_bytes": int(binary_size_bytes.decode("utf-8")), + "$git_commit_time": git_commit_time_unix, + "$git_commit_message": sanitize_str(git_commit_message), + } + + for column in FROM_ENV_COLUMNS: + value = os.environ.get(column.upper(), None) + parameters["$" + column] = sanitize_str(value) + + print("Executing query:\n{}".format(text_query)) + print("With parameters:") + for k, v in parameters.items(): + print("{}: {}".format(k, v)) + + tx.execute(prepared_query, parameters, commit_tx=True) + + +if __name__ == "__main__": + exit(main()) diff --git a/.github/workflows/acceptance_run.yml b/.github/workflows/acceptance_run.yml index 68dbec80efea..0283bf5490b3 100644 --- a/.github/workflows/acceptance_run.yml +++ b/.github/workflows/acceptance_run.yml @@ -25,7 +25,7 @@ on: jobs: main: - name: Build and test ydb/tests/acceptance + name: Build and test ydb/tests/acceptance runs-on: [ self-hosted, "${{ inputs.runner_label || 'auto-provisioned' }}", "${{ format('build-preset-{0}', inputs.build_preset || 'relwithdebinfo') }}" ] steps: - name: Checkout @@ -38,13 +38,18 @@ jobs: with: ssh-private-key: ${{ secrets.SLICE_QA_SSH_PRIVATE_KEY }} + - name: Setup ydb access + uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials + with: + ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} + - name: Build and test uses: ./.github/actions/build_and_test_ya with: build_preset: ${{ inputs.build_preset || 'relwithdebinfo'}} build_target: "ydb/tests/acceptance" increment: false - run_tests: true + run_tests: false test_size: small,medium,large test_type: unittest,py3test,py2test,pytest test_threads: 1 diff --git a/.github/workflows/build_and_test_ya.yml b/.github/workflows/build_and_test_ya.yml index a36a1a947832..bf9639ed9276 100644 --- a/.github/workflows/build_and_test_ya.yml +++ b/.github/workflows/build_and_test_ya.yml @@ -65,6 +65,11 @@ jobs: with: ref: ${{ inputs.commit_sha }} + - name: Setup ydb access + uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials + with: + ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} + - name: Build and test uses: ./.github/actions/build_and_test_ya with: diff --git a/.github/workflows/postcommit_asan.yml b/.github/workflows/postcommit_asan.yml index 5ed8d546aa8c..e90d56053562 100644 --- a/.github/workflows/postcommit_asan.yml +++ b/.github/workflows/postcommit_asan.yml @@ -20,6 +20,10 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 2 + - name: Setup ydb access + uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials + with: + ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} - name: Build and test uses: ./.github/actions/build_and_test_ya with: diff --git a/.github/workflows/postcommit_relwithdebinfo.yml b/.github/workflows/postcommit_relwithdebinfo.yml index 46b018ef8c9d..fb22ad53d610 100644 --- a/.github/workflows/postcommit_relwithdebinfo.yml +++ b/.github/workflows/postcommit_relwithdebinfo.yml @@ -19,6 +19,10 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 2 + - name: Setup ydb access + uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials + with: + ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} - name: Build and test uses: ./.github/actions/build_and_test_ya with: