Add binary size stats

.github/actions/build_ya/action.yml

@@ -145,6 +145,15 @@ runs:
           echo "Build successful." | .github/scripts/tests/comment-pr.py --ok
         fi
 
+    - name: build_stats
+      shell: bash
+      continue-on-error: true
+      run: |
+        set -x
+        export build_preset="${{ inputs.build_preset }}" 
+        python3 -m pip install ydb ydb[yc]
+        python3 .github/scripts/send_build_stats.py
+
     - name: show free space
       if: always()
       shell: bash

.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml

@@ -0,0 +1,19 @@
+name: setup_ci_ydb_service_account_key_file_credentials
+description: setup environment for access to ydb instance in cloud
+inputs:
+  ci_ydb_service_account_key_file_credentials:
+    required: false
+    description: "token for access"
+runs:
+  using: "composite"
+  steps:
+    - name: drop file and set environment variable 
+      if: "${{ inputs.ci_ydb_service_account_key_file_credentials != '' }}"
+      shell: bash
+      run: |
+        set -eu
+        export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=/tmp/ydb_service_account.json
+        cat << EOF > $YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS
+        ${{ inputs.ci_ydb_service_account_key_file_credentials}}
+        EOF
+        echo "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" >> $GITHUB_ENV

.github/scripts/send_build_stats.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+
+import datetime
+import os
+import ydb
+import uuid
+import subprocess
+
+
+YDBD_PATH = "ydb/apps/ydbd/ydbd"
+
+FROM_ENV_COLUMNS = [
+    "github_head_ref",
+    "github_workflow",
+    "github_workflow_ref",
+    "github_sha",
+    "github_repository",
+    "github_event_name",
+    "github_ref_type",
+    "github_ref_name",
+    "github_ref",
+]
+
+STRING_COLUMNS = FROM_ENV_COLUMNS + [
+    "id",
+    "git_commit_message",
+    "binary_path",
+    "build_preset",
+]
+
+DATETIME_COLUMNS = [
+    "git_commit_time",
+]
+
+UINT64_COLUMNS = [
+    "size_bytes",
+    "size_stripped_bytes",
+]
+
+ALL_COLUMNS = STRING_COLUMNS + DATETIME_COLUMNS + UINT64_COLUMNS
+
+
+def sanitize_str(s):
+    # YDB SDK expects bytes for 'String' columns
+    if s is None:
+        s = "N\A"
+    return s.encode("utf-8")
+
+
+def main():
+    if "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" not in os.environ:
+        print("Env variable YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS is missing, skipping")
+        return 1
+
+    with ydb.Driver(
+        endpoint="grpcs://ydb.serverless.yandexcloud.net:2135",
+        database="/ru-central1/b1ggceeul2pkher8vhb6/etn6d1qbals0c29ho4lf",
+        credentials=ydb.credentials_from_env_variables()
+    ) as driver:
+        driver.wait(timeout=10, fail_fast=True)
+        session = ydb.retry_operation_sync(
+            lambda: driver.table_client.session().create()
+        )
+        with session.transaction() as tx:
+            text_query_builder = []
+            for type_ in STRING_COLUMNS:
+                text_query_builder.append("DECLARE ${} as String;".format(type_))
+            for type_ in UINT64_COLUMNS:
+                text_query_builder.append("DECLARE ${} as Uint64;".format(type_))
+            for type_ in DATETIME_COLUMNS:
+                text_query_builder.append("DECLARE ${} as Datetime;".format(type_))
+
+            text_query_builder.append(
+                """INSERT INTO binary_size
+(
+    {}
+)
+VALUES
+(
+    {}                   
+);                         
+""".format(
+                    ", \n    ".join(ALL_COLUMNS),
+                    ", \n    ".join(["$" + column for column in ALL_COLUMNS]),
+                )
+            )
+
+            text_query = "\n".join(text_query_builder)
+
+            prepared_query = session.prepare(text_query)
+
+            binary_size_bytes = subprocess.check_output(
+                ["bash", "-c", "cat {} | wc -c".format(YDBD_PATH)]
+            )
+            binary_size_stripped_bytes = subprocess.check_output(
+                ["bash", "-c", "./ya tool strip {} -o - | wc -c".format(YDBD_PATH)]
+            )
+
+            build_preset = os.environ.get("build_preset", None)
+            github_sha = os.environ.get("GITHUB_SHA", None)
+
+            if github_sha is not None:
+                git_commit_time_bytes = subprocess.check_output(
+                    ["git", "show", "--no-patch", "--format=%cI", github_sha]
+                )
+                git_commit_message_bytes = subprocess.check_output(
+                    ["git", "log", "--format=%s", "-n", "1", github_sha]
+                )
+                git_commit_time = datetime.datetime.fromisoformat(
+                    git_commit_time_bytes.decode("utf-8").strip()
+                )
+                git_commit_message = git_commit_message_bytes.decode("utf-8").strip()
+                git_commit_time_unix = int(git_commit_time.timestamp())
+            else:
+                git_commit_time = None
+                git_commit_message = None
+                git_commit_time_unix = 0
+
+            parameters = {
+                "$id": sanitize_str(str(uuid.uuid4())),
+                "$build_preset": sanitize_str(build_preset),
+                "$binary_path": sanitize_str(YDBD_PATH),
+                "$size_stripped_bytes": int(binary_size_stripped_bytes.decode("utf-8")),
+                "$size_bytes": int(binary_size_bytes.decode("utf-8")),
+                "$git_commit_time": git_commit_time_unix,
+                "$git_commit_message": sanitize_str(git_commit_message),
+            }
+
+            for column in FROM_ENV_COLUMNS:
+                value = os.environ.get(column.upper(), None)
+                parameters["$" + column] = sanitize_str(value)
+
+            print("Executing query:\n{}".format(text_query))
+            print("With parameters:")
+            for k, v in parameters.items():
+                print("{}: {}".format(k, v))
+
+            tx.execute(prepared_query, parameters, commit_tx=True)
+
+
+if __name__ == "__main__":
+    exit(main())

.github/workflows/acceptance_run.yml

@@ -25,7 +25,7 @@ on:
 
 jobs:
   main:
-    name: Build and test ydb/tests/acceptance    
+    name: Build and test ydb/tests/acceptance
     runs-on: [ self-hosted, "${{ inputs.runner_label || 'auto-provisioned' }}", "${{ format('build-preset-{0}', inputs.build_preset || 'relwithdebinfo') }}" ]
     steps:
     - name: Checkout
@@ -38,13 +38,18 @@ jobs:
       with:
         ssh-private-key: ${{ secrets.SLICE_QA_SSH_PRIVATE_KEY }}
 
+    - name: Setup ydb access
+      uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials
+      with:
+        ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }}
+
     - name: Build and test
       uses: ./.github/actions/build_and_test_ya
       with:
         build_preset: ${{ inputs.build_preset || 'relwithdebinfo'}}
         build_target: "ydb/tests/acceptance"
         increment: false
-        run_tests: true
+        run_tests: false
         test_size: small,medium,large
         test_type: unittest,py3test,py2test,pytest
         test_threads: 1

.github/workflows/build_and_test_ya.yml

@@ -65,6 +65,11 @@ jobs:
       with:
         ref: ${{ inputs.commit_sha }}
 
+    - name: Setup ydb access
+      uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials
+      with:
+        ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }}
+
     - name: Build and test
       uses: ./.github/actions/build_and_test_ya
       with:

.github/workflows/postcommit_asan.yml

@@ -20,6 +20,10 @@ jobs:
       uses: actions/checkout@v3
       with:
         fetch-depth: 2
+    - name: Setup ydb access
+      uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials
+      with:
+        ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }}
     - name: Build and test
       uses: ./.github/actions/build_and_test_ya
       with:

.github/workflows/postcommit_relwithdebinfo.yml

@@ -19,6 +19,10 @@ jobs:
       uses: actions/checkout@v3
       with:
         fetch-depth: 2
+    - name: Setup ydb access
+      uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials
+      with:
+        ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }}
     - name: Build and test
       uses: ./.github/actions/build_and_test_ya
       with: