From e2c66651bcff55aae5f0865fd9e88f3f719b0a41 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Mon, 13 May 2024 14:18:49 +0000 Subject: [PATCH 01/35] build analytics --- .github/actions/build_ya/action.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/actions/build_ya/action.yml b/.github/actions/build_ya/action.yml index 2869db1eb766..5c9f38b297ad 100644 --- a/.github/actions/build_ya/action.yml +++ b/.github/actions/build_ya/action.yml @@ -145,6 +145,22 @@ runs: echo "Build successful." | .github/scripts/tests/comment-pr.py --ok fi + - name: build_analytics + shell: bash + run: | + set -x + if [ "${{ steps.build.outputs.status }}" == "failed" ]; then + echo "Build failed, no analytics is send" + else + ydbd_path=ydb/apps/ydbd/ydbd + if [ -e $ydbd_path ]; then + echo "Ydbd exists" + du -shL $ydbd_path + else + echo "No ydbd exists" + fi + fi + - name: show free space if: always() shell: bash From 1963232a351112fd3be55ad5e9afe43acd5d2b6b Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Mon, 13 May 2024 14:24:36 +0000 Subject: [PATCH 02/35] stripped --- .github/actions/build_ya/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/actions/build_ya/action.yml b/.github/actions/build_ya/action.yml index 5c9f38b297ad..17091a4e1b9b 100644 --- a/.github/actions/build_ya/action.yml +++ b/.github/actions/build_ya/action.yml @@ -154,8 +154,12 @@ runs: else ydbd_path=ydb/apps/ydbd/ydbd if [ -e $ydbd_path ]; then + ydbd_stripped_path=ydb/apps/ydbd/ydbd_stripped echo "Ydbd exists" du -shL $ydbd_path + rm -f $ydbd_stripped_path + ya tool strip $ydbd_path $ydbd_stripped_path + du -shL $ydbd_stripped_path else echo "No ydbd exists" fi From 08ab67bb937d043b09242317839bad3355c9f782 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Mon, 13 May 2024 14:25:54 +0000 Subject: [PATCH 03/35] mute tmp --- .github/workflows/acceptance_run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/acceptance_run.yml b/.github/workflows/acceptance_run.yml index 68dbec80efea..00c52f284d33 100644 --- a/.github/workflows/acceptance_run.yml +++ b/.github/workflows/acceptance_run.yml @@ -44,7 +44,7 @@ jobs: build_preset: ${{ inputs.build_preset || 'relwithdebinfo'}} build_target: "ydb/tests/acceptance" increment: false - run_tests: true + run_tests: false test_size: small,medium,large test_type: unittest,py3test,py2test,pytest test_threads: 1 From cb4e219b7d1896efda84096e4e29caf6294112f2 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Mon, 13 May 2024 14:28:09 +0000 Subject: [PATCH 04/35] fix --- .github/actions/build_ya/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build_ya/action.yml b/.github/actions/build_ya/action.yml index 17091a4e1b9b..f48231161ec0 100644 --- a/.github/actions/build_ya/action.yml +++ b/.github/actions/build_ya/action.yml @@ -158,7 +158,7 @@ runs: echo "Ydbd exists" du -shL $ydbd_path rm -f $ydbd_stripped_path - ya tool strip $ydbd_path $ydbd_stripped_path + ya tool strip $ydbd_path -o $ydbd_stripped_path du -shL $ydbd_stripped_path else echo "No ydbd exists" From e05a1d1aaaa0595e07eee78d220cc449680bb1fb Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Mon, 13 May 2024 14:52:13 +0000 Subject: [PATCH 05/35] -DDUMP_LINKER_MAP --- .github/actions/build_ya/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build_ya/action.yml b/.github/actions/build_ya/action.yml index f48231161ec0..d16b72e1987a 100644 --- a/.github/actions/build_ya/action.yml +++ b/.github/actions/build_ya/action.yml @@ -117,7 +117,7 @@ runs: ./ya make -k --build "${build_type}" --force-build-depends -T --stat -DCONSISTENT_DEBUG \ --log-file "$TMP_DIR/ya_log.txt" --evlog-file "$TMP_DIR/ya_evlog.jsonl" \ - --cache-size 512G --link-threads "${{ inputs.link_threads }}" \ + --cache-size 512G --link-threads "${{ inputs.link_threads }}" -DDUMP_LINKER_MAP \ "${extra_params[@]}" |& tee $TMP_DIR/ya_make.log && echo "status=true" >> $GITHUB_OUTPUT || ( RC=$? echo "::debug::ya make RC=$RC" From 5eb5776e7ac7d76dc52353ad985355d0027b694f Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Mon, 13 May 2024 18:19:11 +0000 Subject: [PATCH 06/35] buidl stats --- .github/actions/build_ya/action.yml | 7 ++++ .github/scripts/send_build_stats.py | 53 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100755 .github/scripts/send_build_stats.py diff --git a/.github/actions/build_ya/action.yml b/.github/actions/build_ya/action.yml index d16b72e1987a..3bf192b1f37b 100644 --- a/.github/actions/build_ya/action.yml +++ b/.github/actions/build_ya/action.yml @@ -149,6 +149,12 @@ runs: shell: bash run: | set -x + cat <<"EOF" > $TMP_DIR/YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS + $CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS + EOF + export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$TMP_DIR/YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS + python3 -m pip install ydb ydb[yc] + if [ "${{ steps.build.outputs.status }}" == "failed" ]; then echo "Build failed, no analytics is send" else @@ -160,6 +166,7 @@ runs: rm -f $ydbd_stripped_path ya tool strip $ydbd_path -o $ydbd_stripped_path du -shL $ydbd_stripped_path + python .github/scripts/send_build_stats.py else echo "No ydbd exists" fi diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py new file mode 100755 index 000000000000..ea64237b8606 --- /dev/null +++ b/.github/scripts/send_build_stats.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import os +import ydb + +""" +CREATE TABLE binary_size ( + id String NOT NULL, + github_head_ref String, + github_workflow String, + github_workflow_ref String, + + github_sha String, + github_repository String, + + github_event_name String, + + github_ref_type String, + github_ref_name String, + github_ref String, + + binary_path String, + size_bytes Uint64, + size_stripped_bytes Uint64, + build_type String, + + PRIMARY KEY (id) +) +""" + +def main(): + # token = open("/home/maxim-yurchuk/.yc_token").read().strip() + # os.environ["IAM_TOKEN"] = "/home/maxim-yurchuk/.iam" + + # os.environ["YDB_ACCESS_TOKEN_CREDENTIALS"] = "YDB_ACCESS_TOKEN_CREDENTIALS" + + # token = open("/home/maxim-yurchuk/.iam").read().strip() + # os.environ["YDB_ACCESS_TOKEN_CREDENTIALS"] = token + + + # os.environ["YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"] = "/home/maxim-yurchuk/.sa_token" + + with ydb.Driver( + endpoint="grpcs://ydb.serverless.yandexcloud.net:2135", + database="/ru-central1/b1ggceeul2pkher8vhb6/etn6d1qbals0c29ho4lf", + # credentials=ydb.credentials.AccessTokenCredentials(token), + credentials=ydb.credentials_from_env_variables() + ) as driver: + driver.wait(timeout=10, fail_fast=True) + print("Hi") + +if __name__ == "__main__": + main() From 4fbf1578ee7316e3503fdc3a922e94996269af9a Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Mon, 13 May 2024 18:26:00 +0000 Subject: [PATCH 07/35] pass secret --- .github/workflows/acceptance_run.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/acceptance_run.yml b/.github/workflows/acceptance_run.yml index 00c52f284d33..c8c101cf7e3e 100644 --- a/.github/workflows/acceptance_run.yml +++ b/.github/workflows/acceptance_run.yml @@ -49,7 +49,7 @@ jobs: test_type: unittest,py3test,py2test,pytest test_threads: 1 put_build_results_to_cache: true - secs: ${{ format('{{"TESTMO_TOKEN":"{0}","AWS_KEY_ID":"{1}","AWS_KEY_VALUE":"{2}","REMOTE_CACHE_USERNAME":"{3}","REMOTE_CACHE_PASSWORD":"{4}"}}', - secrets.TESTMO_TOKEN, secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD ) }} + secs: ${{ format('{{"TESTMO_TOKEN":"{0}","AWS_KEY_ID":"{1}","AWS_KEY_VALUE":"{2}","REMOTE_CACHE_USERNAME":"{3}","REMOTE_CACHE_PASSWORD":"{4}", "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS":{5}}}', + secrets.TESTMO_TOKEN, secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD, secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS ) }} vars: ${{ format('{{"AWS_BUCKET":"{0}","AWS_ENDPOINT":"{1}","REMOTE_CACHE_URL":"{2}","TESTMO_URL":"{3}","TESTMO_PROJECT_ID":"{4}"}}', vars.AWS_BUCKET, vars.AWS_ENDPOINT, vars.REMOTE_CACHE_URL_YA, vars.TESTMO_URL, vars.TESTMO_PROJECT_ID ) }} From 8e0913b6ef241c033238105d415bb0209fc10211 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 08:57:29 +0000 Subject: [PATCH 08/35] try fix --- .github/actions/build_ya/action.yml | 6 +++--- .github/workflows/acceptance_run.yml | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/actions/build_ya/action.yml b/.github/actions/build_ya/action.yml index 3bf192b1f37b..97185864cefe 100644 --- a/.github/actions/build_ya/action.yml +++ b/.github/actions/build_ya/action.yml @@ -149,9 +149,9 @@ runs: shell: bash run: | set -x - cat <<"EOF" > $TMP_DIR/YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS - $CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS - EOF + # cat <<"EOF" > $TMP_DIR/YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS + # $CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS + # EOF export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$TMP_DIR/YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS python3 -m pip install ydb ydb[yc] diff --git a/.github/workflows/acceptance_run.yml b/.github/workflows/acceptance_run.yml index c8c101cf7e3e..333911eeddb8 100644 --- a/.github/workflows/acceptance_run.yml +++ b/.github/workflows/acceptance_run.yml @@ -38,6 +38,13 @@ jobs: with: ssh-private-key: ${{ secrets.SLICE_QA_SSH_PRIVATE_KEY }} + - name: Write secret to file + shell: bash + run: | + cat << EOF > /tmp/bla.json + ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} + EOF + - name: Build and test uses: ./.github/actions/build_and_test_ya with: From b03069f4e6107f660cec7941c2c876830d5f2404 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 09:01:20 +0000 Subject: [PATCH 09/35] new --- .github/workflows/acceptance_run.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/acceptance_run.yml b/.github/workflows/acceptance_run.yml index 333911eeddb8..dcd468eef86d 100644 --- a/.github/workflows/acceptance_run.yml +++ b/.github/workflows/acceptance_run.yml @@ -41,9 +41,11 @@ jobs: - name: Write secret to file shell: bash run: | - cat << EOF > /tmp/bla.json + export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=/tmp/ydb_service_account.json + cat << EOF > $YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} EOF + echo "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" >> $GITHUB_ENV - name: Build and test uses: ./.github/actions/build_and_test_ya From a4b4d18120da9c2ee5bbb5242faf3f692cf31bc0 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 09:04:28 +0000 Subject: [PATCH 10/35] fix --- .github/actions/build_ya/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build_ya/action.yml b/.github/actions/build_ya/action.yml index 97185864cefe..7327a97fe4f7 100644 --- a/.github/actions/build_ya/action.yml +++ b/.github/actions/build_ya/action.yml @@ -152,7 +152,7 @@ runs: # cat <<"EOF" > $TMP_DIR/YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS # $CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS # EOF - export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$TMP_DIR/YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS + # export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$TMP_DIR/YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS python3 -m pip install ydb ydb[yc] if [ "${{ steps.build.outputs.status }}" == "failed" ]; then From 6c876c79e36033c5330c73871c7d19bf72d4b4a5 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 10:44:03 +0000 Subject: [PATCH 11/35] stats --- .github/scripts/send_build_stats.py | 97 ++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py index ea64237b8606..b5ad81d058e4 100755 --- a/.github/scripts/send_build_stats.py +++ b/.github/scripts/send_build_stats.py @@ -2,6 +2,7 @@ import os import ydb +import uuid """ CREATE TABLE binary_size ( @@ -28,6 +29,35 @@ ) """ +from_env_columns = [ + "github_head_ref", + "github_workflow", + "github_workflow_ref", + + "github_sha", + "github_repository", + + "github_event_name", + + "github_ref_type", + "github_ref_name", + "github_ref", +] + +string_types = from_env_columns + [ + "id", + + + "binary_path", + "build_type", +] + +uint64_types = [ + "size_bytes", + "size_stripped_bytes", +] + + def main(): # token = open("/home/maxim-yurchuk/.yc_token").read().strip() # os.environ["IAM_TOKEN"] = "/home/maxim-yurchuk/.iam" @@ -38,7 +68,7 @@ def main(): # os.environ["YDB_ACCESS_TOKEN_CREDENTIALS"] = token - # os.environ["YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"] = "/home/maxim-yurchuk/.sa_token" + os.environ["YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"] = "/tmp/ydb_service_account.json" with ydb.Driver( endpoint="grpcs://ydb.serverless.yandexcloud.net:2135", @@ -47,6 +77,71 @@ def main(): credentials=ydb.credentials_from_env_variables() ) as driver: driver.wait(timeout=10, fail_fast=True) + session = ydb.retry_operation_sync(lambda: driver.table_client.session().create()) + session = driver.table_client.session().create() # TODO REMOVE + with session.transaction() as tx: + text_query_builder = [] + for type_ in string_types: + text_query_builder.append("DECLARE ${} as String;".format(type_)) + for type_ in uint64_types: + text_query_builder.append("DECLARE ${} as Uint64;".format(type_)) + + # text_query_builder.append("DECLARE $v as Uint64;") + # text_query_builder.append("INSERT $v;") + + + text_query_builder.append("""INSERT INTO binary_size + ( + {} + ) + VALUES + ( + {} + ); + """.format(", ".join(string_types + uint64_types), ", ".join(["$" + column for column in string_types + uint64_types]))) + + text_query = "".join(text_query_builder) + + prepared_query = session.prepare(text_query) + + parameters = { + "$id": uuid.uuid4().bytes, + "$build_type": b"sample", + "$binary_path": b"sample", + "$size_stripped_bytes": 123, + "$size_bytes": 123, + } + + for column in from_env_columns: + value = os.environ.get(column.upper(), "N/A") + value_bytes = value.encode("utf-8") + parameters["$" + column] = value_bytes + + result_sets = tx.execute(prepared_query, parameters, commit_tx=True) + + + for result_set in result_sets: + print(result_set.rows) + # assert len(result_set.rows) == 1 + # assert result_set.rows[0]["column0"] == result + + + """ + with ydb.SessionPool(driver) as pool: + def callee(session): + res = session.transaction().execute( + "SELECT $seasonsData", + { + "$seriesData": 1, + "$seasonsData": "2", + "$episodesData": 3, + }, + commit_tx=True, + ) + print(res) + + return pool.retry_operation_sync(callee) + """ print("Hi") if __name__ == "__main__": From 540606ed8ab4aa7b85b66c2685c28e90cee7f7da Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 11:17:21 +0000 Subject: [PATCH 12/35] binary size --- .github/scripts/send_build_stats.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py index b5ad81d058e4..512ec662e4b1 100755 --- a/.github/scripts/send_build_stats.py +++ b/.github/scripts/send_build_stats.py @@ -3,6 +3,7 @@ import os import ydb import uuid +import subprocess """ CREATE TABLE binary_size ( @@ -29,6 +30,8 @@ ) """ +YDBD_PATH = "ydb/apps/ydbd/ydbd" + from_env_columns = [ "github_head_ref", "github_workflow", @@ -86,10 +89,6 @@ def main(): for type_ in uint64_types: text_query_builder.append("DECLARE ${} as Uint64;".format(type_)) - # text_query_builder.append("DECLARE $v as Uint64;") - # text_query_builder.append("INSERT $v;") - - text_query_builder.append("""INSERT INTO binary_size ( {} @@ -103,13 +102,17 @@ def main(): text_query = "".join(text_query_builder) prepared_query = session.prepare(text_query) + + binary_size_str = subprocess.check_output(["bash", "-c", "cat {} | wc -c".format(YDBD_PATH)]) + binary_size_stripped_str = subprocess.check_output(["bash", "-c", "./ya tool strip {} -o - | wc -c".format(YDBD_PATH)]) + parameters = { "$id": uuid.uuid4().bytes, "$build_type": b"sample", "$binary_path": b"sample", - "$size_stripped_bytes": 123, - "$size_bytes": 123, + "$size_stripped_bytes": int(binary_size_str), + "$size_bytes": (binary_size_stripped_str), } for column in from_env_columns: From d39af2475c61107f02eb739fa5360cfd6799b9cf Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 11:20:25 +0000 Subject: [PATCH 13/35] fix --- .github/scripts/send_build_stats.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py index 512ec662e4b1..294d70be16b7 100755 --- a/.github/scripts/send_build_stats.py +++ b/.github/scripts/send_build_stats.py @@ -103,16 +103,16 @@ def main(): prepared_query = session.prepare(text_query) - binary_size_str = subprocess.check_output(["bash", "-c", "cat {} | wc -c".format(YDBD_PATH)]) - binary_size_stripped_str = subprocess.check_output(["bash", "-c", "./ya tool strip {} -o - | wc -c".format(YDBD_PATH)]) + binary_size_bytes = subprocess.check_output(["bash", "-c", "cat {} | wc -c".format(YDBD_PATH)]) + binary_size_stripped_bytes = subprocess.check_output(["bash", "-c", "./ya tool strip {} -o - | wc -c".format(YDBD_PATH)]) parameters = { "$id": uuid.uuid4().bytes, "$build_type": b"sample", "$binary_path": b"sample", - "$size_stripped_bytes": int(binary_size_str), - "$size_bytes": (binary_size_stripped_str), + "$size_stripped_bytes": int(binary_size_bytes.decode("utf-8")), + "$size_bytes": (binary_size_stripped_bytes.decode("utf-8")), } for column in from_env_columns: From a5f996f4d3260d81d10980f691fda6dd568ac46d Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 11:22:11 +0000 Subject: [PATCH 14/35] fix --- .github/scripts/send_build_stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py index 294d70be16b7..be3da73c8996 100755 --- a/.github/scripts/send_build_stats.py +++ b/.github/scripts/send_build_stats.py @@ -112,7 +112,7 @@ def main(): "$build_type": b"sample", "$binary_path": b"sample", "$size_stripped_bytes": int(binary_size_bytes.decode("utf-8")), - "$size_bytes": (binary_size_stripped_bytes.decode("utf-8")), + "$size_bytes": int(binary_size_stripped_bytes.decode("utf-8")), } for column in from_env_columns: From 614b46d7ed94e4e163b86ded5390bc0c30cfee5a Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 11:29:20 +0000 Subject: [PATCH 15/35] fix --- .github/scripts/send_build_stats.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py index be3da73c8996..a36f84e01915 100755 --- a/.github/scripts/send_build_stats.py +++ b/.github/scripts/send_build_stats.py @@ -106,11 +106,11 @@ def main(): binary_size_bytes = subprocess.check_output(["bash", "-c", "cat {} | wc -c".format(YDBD_PATH)]) binary_size_stripped_bytes = subprocess.check_output(["bash", "-c", "./ya tool strip {} -o - | wc -c".format(YDBD_PATH)]) - + build_type = os.environ.get("build_type", "N/A").encode("utf-8") parameters = { "$id": uuid.uuid4().bytes, - "$build_type": b"sample", - "$binary_path": b"sample", + "$build_type": build_type, + "$binary_path": YDBD_PATH, "$size_stripped_bytes": int(binary_size_bytes.decode("utf-8")), "$size_bytes": int(binary_size_stripped_bytes.decode("utf-8")), } From a3e5ca552771037f7cc9830167b69bce289b98f6 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 11:40:31 +0000 Subject: [PATCH 16/35] fixes --- .github/scripts/send_build_stats.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py index a36f84e01915..a029159e1775 100755 --- a/.github/scripts/send_build_stats.py +++ b/.github/scripts/send_build_stats.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import json import os import ydb import uuid @@ -90,16 +91,16 @@ def main(): text_query_builder.append("DECLARE ${} as Uint64;".format(type_)) text_query_builder.append("""INSERT INTO binary_size - ( - {} - ) - VALUES - ( - {} - ); - """.format(", ".join(string_types + uint64_types), ", ".join(["$" + column for column in string_types + uint64_types]))) +( + {} +) +VALUES +( + {} +); +""".format(", \n ".join(string_types + uint64_types), ", \n ".join(["$" + column for column in string_types + uint64_types]))) - text_query = "".join(text_query_builder) + text_query = "\n".join(text_query_builder) prepared_query = session.prepare(text_query) @@ -108,9 +109,9 @@ def main(): build_type = os.environ.get("build_type", "N/A").encode("utf-8") parameters = { - "$id": uuid.uuid4().bytes, + "$id": str(uuid.uuid4()).encode("utf-8"), "$build_type": build_type, - "$binary_path": YDBD_PATH, + "$binary_path": YDBD_PATH.encode("utf-8"), "$size_stripped_bytes": int(binary_size_bytes.decode("utf-8")), "$size_bytes": int(binary_size_stripped_bytes.decode("utf-8")), } @@ -120,6 +121,11 @@ def main(): value_bytes = value.encode("utf-8") parameters["$" + column] = value_bytes + print("Executing query:\n{}".format(text_query)) + print("With parameters:") + for k, v in parameters.items(): + print("{}: {}".format(k, v)) + result_sets = tx.execute(prepared_query, parameters, commit_tx=True) From 0c1b3e36cf57af64cb0bd3eafda5e70417862af5 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 13:20:16 +0000 Subject: [PATCH 17/35] fix --- .github/scripts/send_build_stats.py | 139 +++++++++++++++------------- 1 file changed, 75 insertions(+), 64 deletions(-) diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py index a029159e1775..f2f396a037d6 100755 --- a/.github/scripts/send_build_stats.py +++ b/.github/scripts/send_build_stats.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -import json +import datetime import os -import ydb +import ydb import uuid import subprocess @@ -37,60 +37,67 @@ "github_head_ref", "github_workflow", "github_workflow_ref", - "github_sha", "github_repository", - "github_event_name", - "github_ref_type", "github_ref_name", "github_ref", ] -string_types = from_env_columns + [ +string_columns = from_env_columns + [ "id", - - + "git_commit_message", "binary_path", "build_type", ] -uint64_types = [ +datetime_colums = [ + "git_commit_time", +] + +uint64_columns = [ "size_bytes", "size_stripped_bytes", ] +all_columns = string_columns + datetime_colums + uint64_columns -def main(): - # token = open("/home/maxim-yurchuk/.yc_token").read().strip() - # os.environ["IAM_TOKEN"] = "/home/maxim-yurchuk/.iam" - - # os.environ["YDB_ACCESS_TOKEN_CREDENTIALS"] = "YDB_ACCESS_TOKEN_CREDENTIALS" - - # token = open("/home/maxim-yurchuk/.iam").read().strip() - # os.environ["YDB_ACCESS_TOKEN_CREDENTIALS"] = token +def sanitize_str(s): + # YDB SDK expects bytes for 'String' columns + if s is None: + s = "N\A" + return s.encode("utf-8") + + +def main(): + """ + os.environ[ + "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" + ] = "/tmp/ydb_service_account.json" + """ - os.environ["YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"] = "/tmp/ydb_service_account.json" - with ydb.Driver( endpoint="grpcs://ydb.serverless.yandexcloud.net:2135", database="/ru-central1/b1ggceeul2pkher8vhb6/etn6d1qbals0c29ho4lf", - # credentials=ydb.credentials.AccessTokenCredentials(token), - credentials=ydb.credentials_from_env_variables() + credentials=ydb.credentials_from_env_variables(), ) as driver: driver.wait(timeout=10, fail_fast=True) - session = ydb.retry_operation_sync(lambda: driver.table_client.session().create()) - session = driver.table_client.session().create() # TODO REMOVE + session = ydb.retry_operation_sync( + lambda: driver.table_client.session().create() + ) with session.transaction() as tx: text_query_builder = [] - for type_ in string_types: + for type_ in string_columns: text_query_builder.append("DECLARE ${} as String;".format(type_)) - for type_ in uint64_types: + for type_ in uint64_columns: text_query_builder.append("DECLARE ${} as Uint64;".format(type_)) - - text_query_builder.append("""INSERT INTO binary_size + for type_ in datetime_colums: + text_query_builder.append("DECLARE ${} as Datetime;".format(type_)) + + text_query_builder.append( + """INSERT INTO binary_size ( {} ) @@ -98,60 +105,64 @@ def main(): ( {} ); -""".format(", \n ".join(string_types + uint64_types), ", \n ".join(["$" + column for column in string_types + uint64_types]))) +""".format( + ", \n ".join(all_columns), + ", \n ".join(["$" + column for column in all_columns]), + ) + ) text_query = "\n".join(text_query_builder) prepared_query = session.prepare(text_query) - - binary_size_bytes = subprocess.check_output(["bash", "-c", "cat {} | wc -c".format(YDBD_PATH)]) - binary_size_stripped_bytes = subprocess.check_output(["bash", "-c", "./ya tool strip {} -o - | wc -c".format(YDBD_PATH)]) - build_type = os.environ.get("build_type", "N/A").encode("utf-8") + binary_size_bytes = subprocess.check_output( + ["bash", "-c", "cat {} | wc -c".format(YDBD_PATH)] + ) + binary_size_stripped_bytes = subprocess.check_output( + ["bash", "-c", "./ya tool strip {} -o - | wc -c".format(YDBD_PATH)] + ) + + build_type = os.environ.get("build_type", None) + github_sha = os.environ.get("GITHUB_SHA", None) + + if github_sha is not None: + git_commit_time_bytes = subprocess.check_output( + ["git", "show", "--no-patch", "--format=%cI", github_sha] + ) + git_commit_message_bytes = subprocess.check_output( + ["git", "log", "--format=%B", "-n", "1", github_sha] + ) + git_commit_time = datetime.datetime.fromisoformat( + git_commit_time_bytes.decode("utf-8").strip() + ) + git_commit_message = git_commit_message_bytes.decode("utf-8") + git_commit_time_unix = int(git_commit_time.timestamp()) + else: + git_commit_time = None + git_commit_message = None + git_commit_time_unix = 0 + parameters = { - "$id": str(uuid.uuid4()).encode("utf-8"), - "$build_type": build_type, - "$binary_path": YDBD_PATH.encode("utf-8"), + "$id": sanitize_str(str(uuid.uuid4())), + "$build_type": sanitize_str(build_type), + "$binary_path": sanitize_str(YDBD_PATH), "$size_stripped_bytes": int(binary_size_bytes.decode("utf-8")), "$size_bytes": int(binary_size_stripped_bytes.decode("utf-8")), + "$git_commit_time": git_commit_time_unix, + "$git_commit_message": sanitize_str(git_commit_message), } for column in from_env_columns: - value = os.environ.get(column.upper(), "N/A") - value_bytes = value.encode("utf-8") - parameters["$" + column] = value_bytes - + value = os.environ.get(column.upper(), None) + parameters["$" + column] = sanitize_str(value) + print("Executing query:\n{}".format(text_query)) print("With parameters:") for k, v in parameters.items(): print("{}: {}".format(k, v)) result_sets = tx.execute(prepared_query, parameters, commit_tx=True) - - - for result_set in result_sets: - print(result_set.rows) - # assert len(result_set.rows) == 1 - # assert result_set.rows[0]["column0"] == result - - - """ - with ydb.SessionPool(driver) as pool: - def callee(session): - res = session.transaction().execute( - "SELECT $seasonsData", - { - "$seriesData": 1, - "$seasonsData": "2", - "$episodesData": 3, - }, - commit_tx=True, - ) - print(res) - return pool.retry_operation_sync(callee) - """ - print("Hi") - + if __name__ == "__main__": main() From f66399bc9e77544e9ff51b6d6fa142a75249fc47 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 13:39:59 +0000 Subject: [PATCH 18/35] fixes --- .github/actions/build_ya/action.yml | 26 ++++---------------------- .github/scripts/send_build_stats.py | 14 ++++---------- 2 files changed, 8 insertions(+), 32 deletions(-) diff --git a/.github/actions/build_ya/action.yml b/.github/actions/build_ya/action.yml index 7327a97fe4f7..c24e050c1d62 100644 --- a/.github/actions/build_ya/action.yml +++ b/.github/actions/build_ya/action.yml @@ -145,32 +145,14 @@ runs: echo "Build successful." | .github/scripts/tests/comment-pr.py --ok fi - - name: build_analytics + - name: build_stats shell: bash run: | set -x - # cat <<"EOF" > $TMP_DIR/YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS - # $CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS - # EOF - # export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$TMP_DIR/YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS + set +e # do not fail build if script failed (yet?) + export build_preset="${{ inputs.build_preset }}" python3 -m pip install ydb ydb[yc] - - if [ "${{ steps.build.outputs.status }}" == "failed" ]; then - echo "Build failed, no analytics is send" - else - ydbd_path=ydb/apps/ydbd/ydbd - if [ -e $ydbd_path ]; then - ydbd_stripped_path=ydb/apps/ydbd/ydbd_stripped - echo "Ydbd exists" - du -shL $ydbd_path - rm -f $ydbd_stripped_path - ya tool strip $ydbd_path -o $ydbd_stripped_path - du -shL $ydbd_stripped_path - python .github/scripts/send_build_stats.py - else - echo "No ydbd exists" - fi - fi + python3 .github/scripts/send_build_stats.py - name: show free space if: always() diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py index f2f396a037d6..5870c6e61071 100755 --- a/.github/scripts/send_build_stats.py +++ b/.github/scripts/send_build_stats.py @@ -49,7 +49,7 @@ "id", "git_commit_message", "binary_path", - "build_type", + "build_preset", ] datetime_colums = [ @@ -72,16 +72,10 @@ def sanitize_str(s): def main(): - """ - os.environ[ - "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" - ] = "/tmp/ydb_service_account.json" - """ - with ydb.Driver( endpoint="grpcs://ydb.serverless.yandexcloud.net:2135", database="/ru-central1/b1ggceeul2pkher8vhb6/etn6d1qbals0c29ho4lf", - credentials=ydb.credentials_from_env_variables(), + credentials=ydb.credentials_from_env_variables() # see YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS, ) as driver: driver.wait(timeout=10, fail_fast=True) session = ydb.retry_operation_sync( @@ -122,7 +116,7 @@ def main(): ["bash", "-c", "./ya tool strip {} -o - | wc -c".format(YDBD_PATH)] ) - build_type = os.environ.get("build_type", None) + build_preset = os.environ.get("build_preset", None) github_sha = os.environ.get("GITHUB_SHA", None) if github_sha is not None: @@ -144,7 +138,7 @@ def main(): parameters = { "$id": sanitize_str(str(uuid.uuid4())), - "$build_type": sanitize_str(build_type), + "$build_preset": sanitize_str(build_preset), "$binary_path": sanitize_str(YDBD_PATH), "$size_stripped_bytes": int(binary_size_bytes.decode("utf-8")), "$size_bytes": int(binary_size_stripped_bytes.decode("utf-8")), From 8308d1944fcdd7d9c834e23cfdc4ee1ba59a2e73 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 14:00:53 +0000 Subject: [PATCH 19/35] common --- .../action.yml | 17 +++++++++++++++++ .github/workflows/acceptance_run.yml | 12 ++++-------- 2 files changed, 21 insertions(+), 8 deletions(-) create mode 100644 .github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml diff --git a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml new file mode 100644 index 000000000000..03401e2f0f81 --- /dev/null +++ b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml @@ -0,0 +1,17 @@ +name: setup_ci_ydb_service_account_key_file_credentials +description: setup environment for access to ydb instance in cloud +inputs: + ci_ydb_service_account_key_file_credentials: + required: true + description: "token for access" +runs: + steps: + - name: drop file and set environment variable + shell: bash + run: | + set -eu + export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=/tmp/ydb_service_account.json + cat << EOF > $YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS + ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} + EOF + echo "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" >> $GITHUB_ENV \ No newline at end of file diff --git a/.github/workflows/acceptance_run.yml b/.github/workflows/acceptance_run.yml index dcd468eef86d..f72ed8c0323e 100644 --- a/.github/workflows/acceptance_run.yml +++ b/.github/workflows/acceptance_run.yml @@ -38,14 +38,10 @@ jobs: with: ssh-private-key: ${{ secrets.SLICE_QA_SSH_PRIVATE_KEY }} - - name: Write secret to file - shell: bash - run: | - export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=/tmp/ydb_service_account.json - cat << EOF > $YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS - ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} - EOF - echo "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" >> $GITHUB_ENV + - name: Setup ydb access + uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials + with: + ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} - name: Build and test uses: ./.github/actions/build_and_test_ya From c0b0a7a697121f72d58fcb6b2b62421457b0ed75 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 14:02:19 +0000 Subject: [PATCH 20/35] fix --- .../action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml index 03401e2f0f81..95ecf3703ad4 100644 --- a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml +++ b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml @@ -12,6 +12,6 @@ runs: set -eu export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=/tmp/ydb_service_account.json cat << EOF > $YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS - ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} + ${{ inputs.ci_ydb_service_account_key_file_credentials }} EOF echo "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" >> $GITHUB_ENV \ No newline at end of file From 9fc7ff7ea27be8034f9a3d9d689dd3948b3e4a0e Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 14:06:30 +0000 Subject: [PATCH 21/35] fix --- .../setup_ci_ydb_service_account_key_file_credentials/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml index 95ecf3703ad4..ebb57f013064 100644 --- a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml +++ b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml @@ -5,6 +5,7 @@ inputs: required: true description: "token for access" runs: + using: "composite" steps: - name: drop file and set environment variable shell: bash From efbcbed1f694dae5a93ee37a38632fa6ad71eb40 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 14:16:09 +0000 Subject: [PATCH 22/35] All wf --- .github/scripts/send_build_stats.py | 34 ++++--------------- .github/workflows/acceptance_run.yml | 2 +- .github/workflows/build_and_test_ya.yml | 5 +++ .github/workflows/postcommit_asan.yml | 4 +++ .../workflows/postcommit_relwithdebinfo.yml | 4 +++ .github/workflows/pr_check.yml | 4 +++ 6 files changed, 25 insertions(+), 28 deletions(-) diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py index 5870c6e61071..8a89b4cd32dc 100755 --- a/.github/scripts/send_build_stats.py +++ b/.github/scripts/send_build_stats.py @@ -6,30 +6,6 @@ import uuid import subprocess -""" -CREATE TABLE binary_size ( - id String NOT NULL, - github_head_ref String, - github_workflow String, - github_workflow_ref String, - - github_sha String, - github_repository String, - - github_event_name String, - - github_ref_type String, - github_ref_name String, - github_ref String, - - binary_path String, - size_bytes Uint64, - size_stripped_bytes Uint64, - build_type String, - - PRIMARY KEY (id) -) -""" YDBD_PATH = "ydb/apps/ydbd/ydbd" @@ -72,10 +48,14 @@ def sanitize_str(s): def main(): + if "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" not in os.environ: + print("Env variable YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS is missing, skipping") + return 1 + with ydb.Driver( endpoint="grpcs://ydb.serverless.yandexcloud.net:2135", database="/ru-central1/b1ggceeul2pkher8vhb6/etn6d1qbals0c29ho4lf", - credentials=ydb.credentials_from_env_variables() # see YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS, + credentials=ydb.credentials_from_env_variables() ) as driver: driver.wait(timeout=10, fail_fast=True) session = ydb.retry_operation_sync( @@ -155,8 +135,8 @@ def main(): for k, v in parameters.items(): print("{}: {}".format(k, v)) - result_sets = tx.execute(prepared_query, parameters, commit_tx=True) + tx.execute(prepared_query, parameters, commit_tx=True) if __name__ == "__main__": - main() + exit(main()) diff --git a/.github/workflows/acceptance_run.yml b/.github/workflows/acceptance_run.yml index f72ed8c0323e..2310205c94b3 100644 --- a/.github/workflows/acceptance_run.yml +++ b/.github/workflows/acceptance_run.yml @@ -25,7 +25,7 @@ on: jobs: main: - name: Build and test ydb/tests/acceptance + name: Build and test ydb/tests/acceptance runs-on: [ self-hosted, "${{ inputs.runner_label || 'auto-provisioned' }}", "${{ format('build-preset-{0}', inputs.build_preset || 'relwithdebinfo') }}" ] steps: - name: Checkout diff --git a/.github/workflows/build_and_test_ya.yml b/.github/workflows/build_and_test_ya.yml index a36a1a947832..bf9639ed9276 100644 --- a/.github/workflows/build_and_test_ya.yml +++ b/.github/workflows/build_and_test_ya.yml @@ -65,6 +65,11 @@ jobs: with: ref: ${{ inputs.commit_sha }} + - name: Setup ydb access + uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials + with: + ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} + - name: Build and test uses: ./.github/actions/build_and_test_ya with: diff --git a/.github/workflows/postcommit_asan.yml b/.github/workflows/postcommit_asan.yml index 5ed8d546aa8c..e90d56053562 100644 --- a/.github/workflows/postcommit_asan.yml +++ b/.github/workflows/postcommit_asan.yml @@ -20,6 +20,10 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 2 + - name: Setup ydb access + uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials + with: + ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} - name: Build and test uses: ./.github/actions/build_and_test_ya with: diff --git a/.github/workflows/postcommit_relwithdebinfo.yml b/.github/workflows/postcommit_relwithdebinfo.yml index 46b018ef8c9d..fb22ad53d610 100644 --- a/.github/workflows/postcommit_relwithdebinfo.yml +++ b/.github/workflows/postcommit_relwithdebinfo.yml @@ -19,6 +19,10 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 2 + - name: Setup ydb access + uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials + with: + ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} - name: Build and test uses: ./.github/actions/build_and_test_ya with: diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml index 88a8eed4fb28..48633d40a1e3 100644 --- a/.github/workflows/pr_check.yml +++ b/.github/workflows/pr_check.yml @@ -206,6 +206,10 @@ jobs: with: ref: ${{ needs.check-running-allowed.outputs.commit_sha }} fetch-depth: 2 + - name: Setup ydb access + uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials + with: + ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} - name: Build and test uses: ./.github/actions/build_and_test_ya with: From dee1dafba49e434e4705f63f92aef54ae60a2ea5 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 14:23:40 +0000 Subject: [PATCH 23/35] fixes should skip this <---- --- .../action.yml | 2 +- .github/scripts/send_build_stats.py | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml index ebb57f013064..4b520a4b0f90 100644 --- a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml +++ b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml @@ -15,4 +15,4 @@ runs: cat << EOF > $YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS ${{ inputs.ci_ydb_service_account_key_file_credentials }} EOF - echo "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" >> $GITHUB_ENV \ No newline at end of file + echo "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" >> $GITHUB_ENV diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py index 8a89b4cd32dc..bdd473f0e96f 100755 --- a/.github/scripts/send_build_stats.py +++ b/.github/scripts/send_build_stats.py @@ -9,7 +9,7 @@ YDBD_PATH = "ydb/apps/ydbd/ydbd" -from_env_columns = [ +FROM_ENV_COLUMNS = [ "github_head_ref", "github_workflow", "github_workflow_ref", @@ -21,23 +21,23 @@ "github_ref", ] -string_columns = from_env_columns + [ +STRING_COLUMNS = FROM_ENV_COLUMNS + [ "id", "git_commit_message", "binary_path", "build_preset", ] -datetime_colums = [ +DATETIME_COLUMNS = [ "git_commit_time", ] -uint64_columns = [ +UINT64_COLUMNS = [ "size_bytes", "size_stripped_bytes", ] -all_columns = string_columns + datetime_colums + uint64_columns +all_columns = STRING_COLUMNS + DATETIME_COLUMNS + UINT64_COLUMNS def sanitize_str(s): @@ -63,11 +63,11 @@ def main(): ) with session.transaction() as tx: text_query_builder = [] - for type_ in string_columns: + for type_ in STRING_COLUMNS: text_query_builder.append("DECLARE ${} as String;".format(type_)) - for type_ in uint64_columns: + for type_ in UINT64_COLUMNS: text_query_builder.append("DECLARE ${} as Uint64;".format(type_)) - for type_ in datetime_colums: + for type_ in DATETIME_COLUMNS: text_query_builder.append("DECLARE ${} as Datetime;".format(type_)) text_query_builder.append( @@ -104,7 +104,7 @@ def main(): ["git", "show", "--no-patch", "--format=%cI", github_sha] ) git_commit_message_bytes = subprocess.check_output( - ["git", "log", "--format=%B", "-n", "1", github_sha] + ["git", "log", "--format=%s", "-n", "1", github_sha] ) git_commit_time = datetime.datetime.fromisoformat( git_commit_time_bytes.decode("utf-8").strip() @@ -126,7 +126,7 @@ def main(): "$git_commit_message": sanitize_str(git_commit_message), } - for column in from_env_columns: + for column in FROM_ENV_COLUMNS: value = os.environ.get(column.upper(), None) parameters["$" + column] = sanitize_str(value) From 339e926a39d3ae990b3542fc07cc415b55550e3c Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 14:26:50 +0000 Subject: [PATCH 24/35] CAPS --- .github/scripts/send_build_stats.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py index bdd473f0e96f..d30b8f3c33a1 100755 --- a/.github/scripts/send_build_stats.py +++ b/.github/scripts/send_build_stats.py @@ -37,7 +37,7 @@ "size_stripped_bytes", ] -all_columns = STRING_COLUMNS + DATETIME_COLUMNS + UINT64_COLUMNS +ALL_COLUMNS = STRING_COLUMNS + DATETIME_COLUMNS + UINT64_COLUMNS def sanitize_str(s): @@ -80,8 +80,8 @@ def main(): {} ); """.format( - ", \n ".join(all_columns), - ", \n ".join(["$" + column for column in all_columns]), + ", \n ".join(ALL_COLUMNS), + ", \n ".join(["$" + column for column in ALL_COLUMNS]), ) ) From 5366b299e6c7845262ea383ad40d0fd1726dfbe5 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 14:29:57 +0000 Subject: [PATCH 25/35] STRIP() !@@@!#!@#! --- .github/scripts/send_build_stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py index d30b8f3c33a1..3b78134263a1 100755 --- a/.github/scripts/send_build_stats.py +++ b/.github/scripts/send_build_stats.py @@ -109,7 +109,7 @@ def main(): git_commit_time = datetime.datetime.fromisoformat( git_commit_time_bytes.decode("utf-8").strip() ) - git_commit_message = git_commit_message_bytes.decode("utf-8") + git_commit_message = git_commit_message_bytes.decode("utf-8").strip() git_commit_time_unix = int(git_commit_time.timestamp()) else: git_commit_time = None From f30ba6f41c622b8b68f32f43a94512b1cd946054 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 14:56:31 +0000 Subject: [PATCH 26/35] fix --- .github/scripts/send_build_stats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/send_build_stats.py b/.github/scripts/send_build_stats.py index 3b78134263a1..5b1958b10af6 100755 --- a/.github/scripts/send_build_stats.py +++ b/.github/scripts/send_build_stats.py @@ -120,8 +120,8 @@ def main(): "$id": sanitize_str(str(uuid.uuid4())), "$build_preset": sanitize_str(build_preset), "$binary_path": sanitize_str(YDBD_PATH), - "$size_stripped_bytes": int(binary_size_bytes.decode("utf-8")), - "$size_bytes": int(binary_size_stripped_bytes.decode("utf-8")), + "$size_stripped_bytes": int(binary_size_stripped_bytes.decode("utf-8")), + "$size_bytes": int(binary_size_bytes.decode("utf-8")), "$git_commit_time": git_commit_time_unix, "$git_commit_message": sanitize_str(git_commit_message), } From 91d89e5c417e95158acdbf535e9418e174207b40 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 14:57:12 +0000 Subject: [PATCH 27/35] no stats --- .github/actions/build_ya/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build_ya/action.yml b/.github/actions/build_ya/action.yml index c24e050c1d62..8da3578e7025 100644 --- a/.github/actions/build_ya/action.yml +++ b/.github/actions/build_ya/action.yml @@ -117,7 +117,7 @@ runs: ./ya make -k --build "${build_type}" --force-build-depends -T --stat -DCONSISTENT_DEBUG \ --log-file "$TMP_DIR/ya_log.txt" --evlog-file "$TMP_DIR/ya_evlog.jsonl" \ - --cache-size 512G --link-threads "${{ inputs.link_threads }}" -DDUMP_LINKER_MAP \ + --cache-size 512G --link-threads "${{ inputs.link_threads }}" \ "${extra_params[@]}" |& tee $TMP_DIR/ya_make.log && echo "status=true" >> $GITHUB_OUTPUT || ( RC=$? echo "::debug::ya make RC=$RC" From 5d13c6d624567698c170677b75c9d34ca42c911f Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 16:15:50 +0000 Subject: [PATCH 28/35] review fixes --- .../action.yml | 2 +- .github/workflows/acceptance_run.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml index 4b520a4b0f90..9a4997e1cb93 100644 --- a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml +++ b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml @@ -13,6 +13,6 @@ runs: set -eu export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=/tmp/ydb_service_account.json cat << EOF > $YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS - ${{ inputs.ci_ydb_service_account_key_file_credentials }} + ${{ inputs.ci_ydb_service_account_key_file_credentials || "undefined"}} EOF echo "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" >> $GITHUB_ENV diff --git a/.github/workflows/acceptance_run.yml b/.github/workflows/acceptance_run.yml index 2310205c94b3..0283bf5490b3 100644 --- a/.github/workflows/acceptance_run.yml +++ b/.github/workflows/acceptance_run.yml @@ -54,7 +54,7 @@ jobs: test_type: unittest,py3test,py2test,pytest test_threads: 1 put_build_results_to_cache: true - secs: ${{ format('{{"TESTMO_TOKEN":"{0}","AWS_KEY_ID":"{1}","AWS_KEY_VALUE":"{2}","REMOTE_CACHE_USERNAME":"{3}","REMOTE_CACHE_PASSWORD":"{4}", "CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS":{5}}}', - secrets.TESTMO_TOKEN, secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD, secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS ) }} + secs: ${{ format('{{"TESTMO_TOKEN":"{0}","AWS_KEY_ID":"{1}","AWS_KEY_VALUE":"{2}","REMOTE_CACHE_USERNAME":"{3}","REMOTE_CACHE_PASSWORD":"{4}"}}', + secrets.TESTMO_TOKEN, secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD ) }} vars: ${{ format('{{"AWS_BUCKET":"{0}","AWS_ENDPOINT":"{1}","REMOTE_CACHE_URL":"{2}","TESTMO_URL":"{3}","TESTMO_PROJECT_ID":"{4}"}}', vars.AWS_BUCKET, vars.AWS_ENDPOINT, vars.REMOTE_CACHE_URL_YA, vars.TESTMO_URL, vars.TESTMO_PROJECT_ID ) }} From 48f6ea5665bf38ea8864f064630cfe3650b637e3 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 16:21:10 +0000 Subject: [PATCH 29/35] fix --- .../action.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml index 9a4997e1cb93..dd7018f224d0 100644 --- a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml +++ b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml @@ -2,8 +2,9 @@ name: setup_ci_ydb_service_account_key_file_credentials description: setup environment for access to ydb instance in cloud inputs: ci_ydb_service_account_key_file_credentials: - required: true + required: false description: "token for access" + default: "not set" runs: using: "composite" steps: @@ -13,6 +14,6 @@ runs: set -eu export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=/tmp/ydb_service_account.json cat << EOF > $YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS - ${{ inputs.ci_ydb_service_account_key_file_credentials || "undefined"}} + ${{ inputs.ci_ydb_service_account_key_file_credentials || "not set"}} EOF echo "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" >> $GITHUB_ENV From 75d3c1cf2ddd3661afa1b57e513b782765d78d4b Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 16:22:27 +0000 Subject: [PATCH 30/35] 123 --- .../action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml index dd7018f224d0..39ac1cf2551c 100644 --- a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml +++ b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml @@ -14,6 +14,6 @@ runs: set -eu export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=/tmp/ydb_service_account.json cat << EOF > $YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS - ${{ inputs.ci_ydb_service_account_key_file_credentials || "not set"}} + ${{ inputs.ci_ydb_service_account_key_file_credentials || 'not set'}} EOF echo "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" >> $GITHUB_ENV From 77a09e7efd1ccc4a0b515fcce59af34215e2bd50 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 16:27:36 +0000 Subject: [PATCH 31/35] try fix --- .../action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml index 39ac1cf2551c..84a4d753af8a 100644 --- a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml +++ b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml @@ -4,16 +4,16 @@ inputs: ci_ydb_service_account_key_file_credentials: required: false description: "token for access" - default: "not set" runs: using: "composite" steps: - name: drop file and set environment variable + if: "${{ inputs.ci_ydb_service_account_key_file_credentials != '' }}" shell: bash run: | set -eu export YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=/tmp/ydb_service_account.json cat << EOF > $YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS - ${{ inputs.ci_ydb_service_account_key_file_credentials || 'not set'}} + ${{ inputs.ci_ydb_service_account_key_file_credentials}} EOF echo "YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS=$YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS" >> $GITHUB_ENV From a533745427290ae360e7387e1e38493642d9c8e1 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 16:28:58 +0000 Subject: [PATCH 32/35] fix --- .../action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml index 84a4d753af8a..5cac1a347915 100644 --- a/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml +++ b/.github/actions/setup_ci_ydb_service_account_key_file_credentials/action.yml @@ -8,7 +8,7 @@ runs: using: "composite" steps: - name: drop file and set environment variable - if: "${{ inputs.ci_ydb_service_account_key_file_credentials != '' }}" + if: "${{ inputs.ci_ydb_service_account_key_file_credentials != '' }}" shell: bash run: | set -eu From 388f2c2486782fc2f660a4efdcbb967356c0ec0e Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 16:32:57 +0000 Subject: [PATCH 33/35] ignore error --- .github/actions/build_ya/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/build_ya/action.yml b/.github/actions/build_ya/action.yml index c24e050c1d62..a7f357c61e99 100644 --- a/.github/actions/build_ya/action.yml +++ b/.github/actions/build_ya/action.yml @@ -147,6 +147,7 @@ runs: - name: build_stats shell: bash + continue-on-error: true run: | set -x set +e # do not fail build if script failed (yet?) From be8bf9dfe1066431346b0dae08230397544ac23f Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 16:35:35 +0000 Subject: [PATCH 34/35] remove from pr --- .github/workflows/pr_check.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml index 48633d40a1e3..88a8eed4fb28 100644 --- a/.github/workflows/pr_check.yml +++ b/.github/workflows/pr_check.yml @@ -206,10 +206,6 @@ jobs: with: ref: ${{ needs.check-running-allowed.outputs.commit_sha }} fetch-depth: 2 - - name: Setup ydb access - uses: ./.github/actions/setup_ci_ydb_service_account_key_file_credentials - with: - ci_ydb_service_account_key_file_credentials: ${{ secrets.CI_YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS }} - name: Build and test uses: ./.github/actions/build_and_test_ya with: From c7b5c97c73382ec530965d27e3940816179392b3 Mon Sep 17 00:00:00 2001 From: Maxim Yurchuk Date: Tue, 14 May 2024 16:41:31 +0000 Subject: [PATCH 35/35] fix --- .github/actions/build_ya/action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/actions/build_ya/action.yml b/.github/actions/build_ya/action.yml index db236aea19d6..96e35551da50 100644 --- a/.github/actions/build_ya/action.yml +++ b/.github/actions/build_ya/action.yml @@ -150,7 +150,6 @@ runs: continue-on-error: true run: | set -x - set +e # do not fail build if script failed (yet?) export build_preset="${{ inputs.build_preset }}" python3 -m pip install ydb ydb[yc] python3 .github/scripts/send_build_stats.py