Skip to content

Commit

Permalink
Merge branch 'ydb-platform:main' into storageui
Browse files Browse the repository at this point in the history
  • Loading branch information
vporyadke authored Jan 30, 2024
2 parents bffd7f6 + d2c589a commit 6ffc1fc
Show file tree
Hide file tree
Showing 3,660 changed files with 125,514 additions and 68,801 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
12 changes: 12 additions & 0 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -1 +1,13 @@
/ydb/core/fq/ @ydb-platform/fq
/ydb/core/public_http/ @ydb-platform/fq

/ydb/docs/ @ydb-platform/docs

/ydb/library/yql/ @ydb-platform/yql
/ydb/library/yql/providers/generic @ydb-platform/fq
/ydb/library/yql/providers/pq @ydb-platform/fq
/ydb/library/yql/providers/s3 @ydb-platform/fq
/ydb/library/yql/providers/solomon @ydb-platform/fq

/ydb/services/fq/ @ydb-platform/fq

1 change: 0 additions & 1 deletion .github/config/muted_functest.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,4 @@ ydb/tests/functional/tenants/test_storage_config.py::TestStorageConfig::*
ydb/tests/functional/tenants/test_tenants.py::*
ydb/tests/functional/audit/*
ydb/tests/functional/clickbench::test.py.test_plans*
ydb/tests/fq/s3*
ydb/tests/fq/yds/test_metrics_cleanup.py::TestCleanup*test_cleanup[v1]
1 change: 0 additions & 1 deletion .github/config/muted_test.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ ydb-tests-functional-kqp-kqp_query_session/KqpQuerySession::NoLocalAttach
ydb-core-blobstorage-ut_blobstorage/VDiskAssimilation::Test
ydb-core-tx-columnshard-ut_schema/TColumnShardTestSchema::ForgetAfterFail
ydb-core-tx-columnshard-ut_schema/TColumnShardTestSchema::RebootForgetAfterFail
ydb-core-kqp-ut-federated_query-generic/*
ydb-library-yql-sql-pg-ut/PgSqlParsingAutoparam::AutoParamValues_DifferentTypes
ydb-core-blobstorage-ut_blobstorage/[6/10]*
ydb/core/blobstorage/ut_blobstorage/Defragmentation::DoesItWork
56 changes: 48 additions & 8 deletions .github/config/muted_ya.txt
Original file line number Diff line number Diff line change
@@ -1,48 +1,88 @@
ydb/core/blobstorage/dsproxy/ut TBlobStorageProxySequenceTest.TestBlock42PutWithChangingSlowDisk
ydb/core/blobstorage/dsproxy/ut_fat TBlobStorageProxyTest.TestBatchedPutRequestDoesNotContainAHugeBlob
ydb/core/blobstorage/pdisk/ut TSectorMap.*
ydb/core/blobstorage/ut_blobstorage CostMetricsGetBlock4Plus2.TestGet4Plus2BlockRequests10000Inflight1BlobSize1000
ydb/core/blobstorage/ut_blobstorage Defragmentation.DoesItWork
ydb/core/blobstorage/ut_blobstorage SpaceCheckForDiskReassign.*
ydb/core/blobstorage/ut_blobstorage VDiskAssimilation.Test
ydb/core/blobstorage/ut_blobstorage [6/10]*
ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk ReadOnlyVDisk.TestStorageLoad
ydb/core/client/ut TClientTest.PromoteFollower
ydb/core/client/ut TFlatTest.AutoSplitMergeQueue
ydb/core/cms/ut_sentinel TSentinelTests.BSControllerCantChangeStatus
ydb/core/persqueue/ut [31/40]*
ydb/core/persqueue/ut TopicSplitMerge.PartitionSplit
ydb/core/persqueue/ut TPersQueueMirrorer.TestBasicRemote
ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient
ydb/core/kafka_proxy/ut KafkaProtocol.CreatePartitionsScenario
ydb/core/kafka_proxy/ut KafkaProtocol.ProduceScenario
ydb/core/kqp/provider/ut KikimrIcGateway.TestLoadBasicSecretValueFromExternalDataSourceMetadata
ydb/core/kqp/ut/federated_query/generic *
ydb/core/kqp/ut/olap *
ydb/core/kqp/ut/pg KqpPg.CreateIndex
ydb/core/kqp/ut/query KqpLimits.QueryReplySize
ydb/core/kqp/ut/scan KqpRequestContext.TraceIdInErrorMessage
ydb/core/kqp/ut/scheme KqpOlapScheme.TenThousandColumns
ydb/core/kqp/ut/scheme KqpScheme.QueryWithAlter
ydb/core/kqp/ut/scheme [44/50]*
ydb/core/kqp/ut/service KqpQueryService.ExecuteQueryPgTableSelect
ydb/core/kqp/ut/service KqpQueryService.QueryOnClosedSession
ydb/core/kqp/ut/service KqpService.CloseSessionsWithLoad
ydb/core/kqp/ut/service [38/50]*
ydb/core/tx/columnshard/ut_schema TColumnShardTestSchema.ForgetAfterFail
ydb/core/tx/columnshard/ut_schema TColumnShardTestSchema.RebootForgetAfterFail
ydb/core/tx/columnshard/engines/ut *
ydb/core/tx/coordinator/ut Coordinator.RestoreTenantConfiguration
ydb/core/tx/datashard/ut_change_exchange Cdc.InitialScanDebezium
ydb/core/tx/replication/ydb_proxy/ut YdbProxyTests.ReadTopic
ydb/core/tx/schemeshard/ut_restore TImportTests.ShouldSucceedOnManyTables
ydb/core/tx/schemeshard/ut_split_merge TSchemeShardSplitBySizeTest.Merge1KShards
ydb/core/tx/tx_proxy/ut_ext_tenant TExtSubDomainTest.CreateTableInsideAndAlterDomainAndTable-AlterDatabaseCreateHiveFirst*
ydb/core/tx/tx_proxy/ut_storage_tenant TStorageTenantTest.RemoveStoragePoolBeforeDroppingTablet
ydb/core/util/ut TCircularOperationQueueTest.ShouldShuffle
ydb/core/viewer/ut Viewer.TabletMerging
ydb/library/actors/http/ut HttpProxy.TooLongHeader
ydb/library/actors/http/ut sole*
ydb/library/yql/providers/generic/connector/tests sole*
ydb/library/yql/sql/pg/ut PgSqlParsingAutoparam.AutoParamValues_DifferentTypes
ydb/library/yql/tests/sql/dq_file/part16 *
ydb/library/yql/tests/sql/dq_file/part18 test.py.test[expr-cast_type_bind-default.txt-Analyze]
ydb/library/yql/tests/sql/dq_file/part* *
ydb/public/sdk/cpp/client/ydb_persqueue_core/ut/with_offset_ranges_mode_ut RetryPolicy.RetryWithBatching
ydb/public/sdk/cpp/client/ydb_topic/ut BasicUsage.WriteRead
ydb/services/datastreams/ut DataStreams.TestPutRecordsWithRead
ydb/services/datastreams/ut DataStreams.TestReservedConsumersMetering
ydb/services/datastreams/ut DataStreams.TestReservedStorageMetering
ydb/services/fq/ut_integration Yq_1.*
ydb/services/persqueue_v1/ut/describes_ut TTopicApiDescribes.DescribeTopic
ydb/services/persqueue_v1/ut TPersQueueTest.DirectRead*
ydb/services/persqueue_v1/ut TPersQueueTest.SetupLockSession
ydb/services/persqueue_v1/ut TPersQueueTest.UpdatePartitionLocation
ydb/services/persqueue_v1/ut [3/10]*
ydb/services/ydb/sdk_sessions_pool_ut YdbSdkSessionsPool.StressTestSync*
ydb/services/ydb/sdk_sessions_ut YdbSdkSessions.TestActiveSessionCountAfterBadSession
ydb/services/ydb/sdk_sessions_ut [6/10]*
ydb/services/ydb/table_split_ut YdbTableSplit.SplitByLoadWithReadsMultipleSplitsWithData
ydb/services/ydb/ut YdbOlapStore.LogPagingAfter-NotNull
ydb/services/ydb/ut YdbQueryService.TestCreateAndAttachSession
ydb/services/ydb/ut YdbScripting.BasicV0
ydb/services/ydb/ut YdbYqlClient.TestYqlSessionClosed
ydb/tests/fq/generic *
ydb/tests/fq/mem_alloc *
ydb/tests/fq/multi_plane test_dispatch.py.TestMapping.test_mapping
ydb/tests/fq/multi_plane sole*
ydb/tests/fq/restarts test_insert_restarts.py.TestS3.test_atomic_upload_commit*
ydb/tests/fq/s3 *
ydb/tests/fq/yds test_metrics_cleanup.py.TestCleanup.test_cleanup[v1]
ydb/tests/fq/yds *
ydb/tests/functional/audit *
ydb/tests/functional/blobstorage test_replication.py.TestReplicationAfterNodesRestart.test_replication[mirror-3-dc]
ydb/tests/functional/blobstorage test_replication.py.TestReplicationAfterNodesRestart.test_replication*
ydb/tests/functional/clickbench test.py.test_plans*
ydb/tests/functional/clickbench test.py.test_run_determentistic[column]
ydb/tests/functional/clickbench test.py.test_run_*column]
ydb/tests/functional/kqp/kqp_indexes ConsistentIndexRead.InteractiveTx
ydb/tests/functional/kqp/kqp_query_session KqpQuerySession.NoLocalAttach
ydb/tests/functional/postgresql test_postgres.py.TestPostgresSuite.test_postgres_suite*
ydb/tests/functional/restarts test_restarts.py.*
ydb/tests/functional/sqs/cloud test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_dlq_mechanics_in_cloud*
ydb/tests/functional/sqs/cloud test_yandex_cloud_queue_counters.py.TestYmqQueueCounters.test_purge_queue_counters
ydb/tests/functional/sqs/common test_queue_counters.py.TestSqsGettingCounters.test_purge_queue_counters
ydb/tests/functional/serverless test_serverless.py.test_database_with_disk_quotas[enable_alter_database_create_hive_first--*]
ydb/tests/functional/sqs/merge_split_common_table/fifo *
ydb/tests/functional/sqs/merge_split_common_table/std *
ydb/tests/functional/tenants test_dynamic_tenants.py.*
ydb/tests/functional/tenants test_storage_config.py.TestStorageConfig.*
ydb/tests/functional/tenants test_tenants.py.*
Expand Down
7 changes: 4 additions & 3 deletions .github/scripts/tests/comment-pr.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ def main():
elif args.fail:
color = 'red'

run_number = int(os.environ.get("GITHUB_RUN_NUMBER"))
build_preset = os.environ["BUILD_PRESET"]

gh = Github(auth=GithubAuth.Token(os.environ["GITHUB_TOKEN"]))
Expand All @@ -31,9 +32,9 @@ def main():
event = json.load(fp)

pr = gh.create_from_raw_data(PullRequest, event["pull_request"])

update_pr_comment_text(pr, build_preset, color, args.text.read().rstrip(), args.rewrite)
update_pr_comment_text(pr, build_preset, run_number, color, args.text.read().rstrip(), args.rewrite)


if __name__ == "__main__":
main()
if os.environ.get('GITHUB_EVENT_NAME', '').startswith('pull_request'):
main()
4 changes: 3 additions & 1 deletion .github/scripts/tests/generate-summary.py
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,9 @@ def main():
else:
color = 'green'

update_pr_comment_text(pr, args.build_preset, color, text='\n'.join(text), rewrite=False)
run_number = int(os.environ.get("GITHUB_RUN_NUMBER"))

update_pr_comment_text(pr, args.build_preset, run_number, color, text='\n'.join(text), rewrite=False)


if __name__ == "__main__":
Expand Down
4 changes: 2 additions & 2 deletions .github/scripts/tests/gh_status.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ def get_platform_name():
return f'{platform.system().lower()}-{platform.machine()}'


def update_pr_comment_text(pr: PullRequest, build_preset: str, color: str, text: str, rewrite: bool):
def update_pr_comment_text(pr: PullRequest, build_preset: str, run_number: int, color: str, text: str, rewrite: bool):
platform_name = get_platform_name()
header = f"<!-- status pr={pr.number}, preset={platform_name}-{build_preset} -->"
header = f"<!-- status pr={pr.number}, preset={platform_name}-{build_preset}, run={run_number} -->"

body = comment = None
for c in pr.get_issue_comments():
Expand Down
15 changes: 7 additions & 8 deletions .github/workflows/build_and_test_ya.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ on:
put_build_results_to_cache:
type: boolean
default: true
commit_sha:
type: string
default: ""
defaults:
run:
shell: bash
Expand All @@ -57,15 +60,11 @@ jobs:
name: Build and test ${{ inputs.build_preset }}
runs-on: [ self-hosted, "${{ inputs.runner_label }}", "${{ inputs.runner_additional_label || inputs.runner_label }}"]
steps:
- name: Checkout PR
uses: actions/checkout@v3
if: github.event.pull_request.head.sha != ''
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Checkout
uses: actions/checkout@v3
if: github.event.pull_request.head.sha == ''

with:
ref: ${{ inputs.commit_sha }}

- name: comment-build-start
if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
shell: bash
Expand All @@ -77,7 +76,7 @@ jobs:
# tricky: we are searching job with name that contains build_preset
check_url=$(curl -s $jobs_url | jq --arg n "$BUILD_PRESET" -r '.jobs[] | select(.name | contains($n)) | .html_url')
echo "Pre-commit [check]($check_url) for ${{ github.event.pull_request.head.sha }} has started." | .github/scripts/tests/comment-pr.py --rewrite
echo "Pre-commit [check]($check_url) for ${{ inputs.commit_sha }} has started." | .github/scripts/tests/comment-pr.py --rewrite
- name: Prepare s3cmd
uses: ./.github/actions/s3cmd
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/build_and_test_ya_provisioned.yml
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,9 @@ on:
put_build_results_to_cache:
type: boolean
default: true
commit_sha:
type: string
default: ""
jobs:
main:
uses: ./.github/workflows/build_and_test_ya.yml
Expand All @@ -107,4 +110,5 @@ jobs:
link_threads: ${{ inputs.link_threads }}
test_threads: ${{ inputs.test_threads }}
put_build_results_to_cache: ${{ inputs.put_build_results_to_cache }}
commit_sha: ${{ inputs.commit_sha }}
secrets: inherit
75 changes: 75 additions & 0 deletions .github/workflows/embedded_ui_refresh.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
on:
repository_dispatch:
types: [embedded_ui_refresh]
workflow_dispatch:
inputs:
asset_name:
description: Release asset name
required: true
default: embedded-ui
repository:
description: Source repository
required: true
default: ydb-platform/ydb-embedded-ui
tag_name:
description: Release tag
required: true

name: Embedded UI Refresh

jobs:
main:
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
env:
ASSET_NAME: ${{ github.event.inputs && github.event.inputs.asset_name || github.event.client_payload.asset_name }}
REPOSITORY: ${{ github.event.inputs && github.event.inputs.repository || github.event.client_payload.repository }}
TAG_NAME: ${{ github.event.inputs && github.event.inputs.tag_name || github.event.client_payload.tag_name }}
TEMP_ASSET_DIR: /tmp/asset
steps:
- uses: actions/checkout@v4
- name: Asset Download
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: gh release download $TAG_NAME --repo $REPOSITORY --pattern $ASSET_NAME.zip --dir $TEMP_ASSET_DIR
- name: Asset Placement
env:
START_POINTER: "# GENERATED MONITORING RESOURCES START"
END_POINTER: "# GENERATED MONITORING RESOURCES END"
TARGET_DIR: ydb/core/viewer/monitoring
YA_MAKE_FILE: ydb/core/viewer/ya.make
run: |
unzip $TEMP_ASSET_DIR/$ASSET_NAME.zip -d $TEMP_ASSET_DIR
rm -rf $TARGET_DIR
mkdir $TARGET_DIR
mv -vf $TEMP_ASSET_DIR/$ASSET_NAME/* $TARGET_DIR
# List of files in the target directory.
NEW_RESOURCES=$(find $TARGET_DIR -type f | sort)
# Current indentation of the start pointer line.
INDENTATION=$(grep -e "$START_POINTER" $YA_MAKE_FILE | perl -lane 's/^(\s+)(.*)+$/$1/e; print')
# Replacing resources list between start and end pointers with saving the current indentation.
perl -0777 -pi -e "s/\s+$START_POINTER.*$END_POINTER/
$INDENTATION$START_POINTER
${INDENTATION}RESOURCE(
$(echo "$NEW_RESOURCES" | perl -e "while (<>) {chomp; print \"$INDENTATION \$_ \$_\\n\";}" | sed -E 's/\//\\\//g')
$INDENTATION)
$INDENTATION$END_POINTER/s" $YA_MAKE_FILE
- name: Pull Request
uses: peter-evans/create-pull-request@v5
with:
token: ${{ secrets.GITHUB_TOKEN }}
commit-message: "build: refresh Embedded UI (${{ env.TAG_NAME }})"
branch: embedded-ui-refresh-${{ env.TAG_NAME }}
delete-branch: true
title: "build: refresh Embedded UI (${{ env.TAG_NAME }})"
body: |
### Embedded UI Refresh
Embedded UI
[${{ env.TAG_NAME }}](https://github.com/${{ env.REPOSITORY }}/releases/tag/${{ env.TAG_NAME }})
([CHANGELOG.md](https://github.com/${{ env.REPOSITORY }}/blob/${{ env.TAG_NAME }}/CHANGELOG.md)).
Loading

0 comments on commit 6ffc1fc

Please sign in to comment.