diff --git a/.github/workflows/cancel_duplicates.yml b/.github/workflows/cancel_duplicates.yml deleted file mode 100644 index a78ebc07fdbe7..0000000000000 --- a/.github/workflows/cancel_duplicates.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Cancel Duplicates -on: - workflow_run: - workflows: - - "Miscellaneous" - types: - - requested - -jobs: - cancel-duplicate-runs: - name: Cancel duplicate workflow runs - runs-on: ubuntu-20.04 - permissions: - actions: write - contents: read - steps: - - name: Check number of queued tasks - id: check_queued - env: - GITHUB_TOKEN: ${{ github.token }} - GITHUB_REPO: ${{ github.repository }} - run: | - get_count() { - echo $(curl -s -H "Authorization: token $GITHUB_TOKEN" \ - "https://api.github.com/repos/$GITHUB_REPO/actions/runs?status=$1" | \ - jq ".total_count") - } - count=$(( `get_count queued` + `get_count in_progress` )) - echo "Found $count unfinished jobs." - echo "::set-output name=count::$count" - - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - if: steps.check_queued.outputs.count >= 20 - uses: actions/checkout@v3 - - - name: Cancel duplicate workflow runs - if: steps.check_queued.outputs.count >= 20 - env: - GITHUB_TOKEN: ${{ github.token }} - GITHUB_REPOSITORY: ${{ github.repository }} - run: | - pip install click requests typing_extensions python-dateutil - python ./scripts/cancel_github_workflows.py diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 90e9f3f2dc941..eee028877aadc 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,6 +13,11 @@ on: schedule: - cron: '0 4 * * *' +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: analyze: name: Analyze diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index ba00b5d3db251..04ca83356dd20 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -10,6 +10,11 @@ on: [pull_request] permissions: contents: read +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: dependency-review: runs-on: ubuntu-latest diff --git a/.github/workflows/docker-ephemeral-env.yml b/.github/workflows/docker-ephemeral-env.yml index bc877ae9ae737..f149e9fea5c19 100644 --- a/.github/workflows/docker-ephemeral-env.yml +++ b/.github/workflows/docker-ephemeral-env.yml @@ -6,6 +6,11 @@ on: types: - completed +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: config: runs-on: "ubuntu-latest" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 43483cf5f99fb..fe8fa4b987b9e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,6 +7,11 @@ on: pull_request: types: [synchronize, opened, reopened, ready_for_review] +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: config: runs-on: "ubuntu-latest" diff --git a/.github/workflows/embedded-sdk-test.yml b/.github/workflows/embedded-sdk-test.yml index 20c7ef5b1b836..99787d9346487 100644 --- a/.github/workflows/embedded-sdk-test.yml +++ b/.github/workflows/embedded-sdk-test.yml @@ -6,6 +6,11 @@ on: - "superset-embedded-sdk/**" types: [synchronize, opened, reopened, ready_for_review] +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: embedded-sdk-test: runs-on: ubuntu-20.04 diff --git a/.github/workflows/pr-lint.yml b/.github/workflows/pr-lint.yml index 5283e138c1a0a..45faef6b44481 100644 --- a/.github/workflows/pr-lint.yml +++ b/.github/workflows/pr-lint.yml @@ -3,11 +3,16 @@ name: PR Lint on: pull_request: # By default, a workflow only runs when a pull_request's activity type is opened, synchronize, or reopened. We - # explicity override here so that PR titles are re-linted when the PR text content is edited. + # explicitly override here so that PR titles are re-linted when the PR text content is edited. # # Possible values: https://help.github.com/en/actions/reference/events-that-trigger-workflows#pull-request-event-pull_request types: [opened, edited, reopened, synchronize] +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: check: runs-on: ubuntu-latest diff --git a/.github/workflows/superset-cli.yml b/.github/workflows/superset-cli.yml index faf6d7d0b21a0..cf344e1b13bb3 100644 --- a/.github/workflows/superset-cli.yml +++ b/.github/workflows/superset-cli.yml @@ -7,6 +7,11 @@ on: pull_request: types: [synchronize, opened, reopened, ready_for_review] +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: test-load-examples: runs-on: ubuntu-20.04 diff --git a/.github/workflows/superset-docs.yml b/.github/workflows/superset-docs.yml index 6a47df4400dd8..942f7874158ed 100644 --- a/.github/workflows/superset-docs.yml +++ b/.github/workflows/superset-docs.yml @@ -8,6 +8,11 @@ on: paths: - "docs/**" +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: config: runs-on: "ubuntu-latest" diff --git a/.github/workflows/superset-e2e.yml b/.github/workflows/superset-e2e.yml index 5eb794a4859f2..566484f8541d4 100644 --- a/.github/workflows/superset-e2e.yml +++ b/.github/workflows/superset-e2e.yml @@ -9,6 +9,11 @@ on: pull_request: types: [synchronize, opened, reopened, ready_for_review] +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: cypress-matrix: runs-on: ubuntu-20.04 diff --git a/.github/workflows/superset-frontend.yml b/.github/workflows/superset-frontend.yml index 35e1a989a4179..e796e99c000eb 100644 --- a/.github/workflows/superset-frontend.yml +++ b/.github/workflows/superset-frontend.yml @@ -8,6 +8,11 @@ on: pull_request: types: [synchronize, opened, reopened, ready_for_review] +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: frontend-build: runs-on: ubuntu-20.04 diff --git a/.github/workflows/superset-helm-lint.yml b/.github/workflows/superset-helm-lint.yml index 4bb888e15d74e..c482f912d8e81 100644 --- a/.github/workflows/superset-helm-lint.yml +++ b/.github/workflows/superset-helm-lint.yml @@ -4,6 +4,11 @@ on: pull_request: types: [opened, edited, reopened, synchronize] +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: lint-test: runs-on: ubuntu-latest diff --git a/.github/workflows/superset-python-integrationtest.yml b/.github/workflows/superset-python-integrationtest.yml index f2caf2de389ad..f8bdc97b9d90b 100644 --- a/.github/workflows/superset-python-integrationtest.yml +++ b/.github/workflows/superset-python-integrationtest.yml @@ -8,6 +8,11 @@ on: pull_request: types: [synchronize, opened, reopened, ready_for_review] +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: test-mysql: runs-on: ubuntu-20.04 diff --git a/.github/workflows/superset-python-misc.yml b/.github/workflows/superset-python-misc.yml index a507467cad311..03da50c6bb852 100644 --- a/.github/workflows/superset-python-misc.yml +++ b/.github/workflows/superset-python-misc.yml @@ -8,6 +8,11 @@ on: pull_request: types: [synchronize, opened, reopened, ready_for_review] +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: python-lint: runs-on: ubuntu-20.04 diff --git a/.github/workflows/superset-python-presto-hive.yml b/.github/workflows/superset-python-presto-hive.yml index 04cdf64d0b2dc..165a854ff8b07 100644 --- a/.github/workflows/superset-python-presto-hive.yml +++ b/.github/workflows/superset-python-presto-hive.yml @@ -8,6 +8,11 @@ on: pull_request: types: [synchronize, opened, reopened, ready_for_review] +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: test-postgres-presto: runs-on: ubuntu-20.04 diff --git a/.github/workflows/superset-python-unittest.yml b/.github/workflows/superset-python-unittest.yml index 5f7639e67ccf7..3027b263685d5 100644 --- a/.github/workflows/superset-python-unittest.yml +++ b/.github/workflows/superset-python-unittest.yml @@ -8,6 +8,11 @@ on: pull_request: types: [synchronize, opened, reopened, ready_for_review] +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: unit-tests: runs-on: ubuntu-20.04 diff --git a/.github/workflows/superset-translations.yml b/.github/workflows/superset-translations.yml index e42a9ac9e72e4..4fb9a79dcb668 100644 --- a/.github/workflows/superset-translations.yml +++ b/.github/workflows/superset-translations.yml @@ -7,6 +7,11 @@ on: pull_request: types: [synchronize, opened, reopened, ready_for_review] +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: frontend-check: runs-on: ubuntu-20.04 diff --git a/.github/workflows/superset-websocket.yml b/.github/workflows/superset-websocket.yml index 73a532cb98108..d62eb8fafe2e8 100644 --- a/.github/workflows/superset-websocket.yml +++ b/.github/workflows/superset-websocket.yml @@ -7,6 +7,11 @@ on: paths: - "superset-websocket/**" +# cancel previous workflow jobs for PRs +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + jobs: app-checks: runs-on: ubuntu-20.04 diff --git a/scripts/cancel_github_workflows.py b/scripts/cancel_github_workflows.py deleted file mode 100755 index 70744c295467b..0000000000000 --- a/scripts/cancel_github_workflows.py +++ /dev/null @@ -1,252 +0,0 @@ -#!/usr/bin/env python3 -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Manually cancel previous GitHub Action workflow runs in queue. - -Example: - # Set up - export GITHUB_TOKEN={{ your personal github access token }} - export GITHUB_REPOSITORY=apache/superset - - # cancel previous jobs for a PR, will even cancel the running ones - ./cancel_github_workflows.py 1042 - - # cancel previous jobs for a branch - ./cancel_github_workflows.py my-branch - - # cancel all jobs of a PR, including the latest runs - ./cancel_github_workflows.py 1024 --include-last -""" -import os -from collections.abc import Iterable, Iterator -from typing import Any, Literal, Optional, Union - -import click -import requests -from click.exceptions import ClickException -from dateutil import parser - -github_token = os.environ.get("GITHUB_TOKEN") -github_repo = os.environ.get("GITHUB_REPOSITORY", "apache/superset") - - -def request( - method: Literal["GET", "POST", "DELETE", "PUT"], endpoint: str, **kwargs: Any -) -> dict[str, Any]: - resp = requests.request( - method, - f"https://api.github.com/{endpoint.lstrip('/')}", - headers={"Authorization": f"Bearer {github_token}"}, - **kwargs, - ).json() - if "message" in resp: - raise ClickException(f"{endpoint} >> {resp['message']} <<") - return resp - - -def list_runs( - repo: str, - params: Optional[dict[str, str]] = None, -) -> Iterator[dict[str, Any]]: - """List all github workflow runs. - Returns: - An iterator that will iterate through all pages of matching runs.""" - if params is None: - params = {} - page = 1 - total_count = 10000 - while page * 100 < total_count: - result = request( - "GET", - f"/repos/{repo}/actions/runs", - params={**params, "per_page": 100, "page": page}, - ) - total_count = result["total_count"] - yield from result["workflow_runs"] - page += 1 - - -def cancel_run(repo: str, run_id: Union[str, int]) -> dict[str, Any]: - return request("POST", f"/repos/{repo}/actions/runs/{run_id}/cancel") - - -def get_pull_request(repo: str, pull_number: Union[str, int]) -> dict[str, Any]: - return request("GET", f"/repos/{repo}/pulls/{pull_number}") - - -def get_runs( - repo: str, - branch: Optional[str] = None, - user: Optional[str] = None, - statuses: Iterable[str] = ("queued", "in_progress"), - events: Iterable[str] = ("pull_request", "push"), -) -> list[dict[str, Any]]: - """Get workflow runs associated with the given branch""" - return [ - item - for event in events - for status in statuses - for item in list_runs(repo, {"event": event, "status": status}) - if (branch is None or (branch == item["head_branch"])) - and (user is None or (user == item["head_repository"]["owner"]["login"])) - ] - - -def print_commit(commit: dict[str, Any], branch: str) -> None: - """Print out commit message for verification""" - indented_message = " \n".join(commit["message"].split("\n")) - date_str = ( - parser.parse(commit["timestamp"]) - .astimezone(tz=None) - .strftime("%a, %d %b %Y %H:%M:%S") - ) - print( - f"""HEAD {commit["id"]} ({branch}) -Author: {commit["author"]["name"]} <{commit["author"]["email"]}> -Date: {date_str} - - {indented_message} -""" - ) - - -@click.command() -@click.option( - "--repo", - default=github_repo, - help="The github repository name. For example, apache/superset.", -) -@click.option( - "--event", - type=click.Choice(["pull_request", "push", "issue"]), - default=["pull_request", "push"], - show_default=True, - multiple=True, -) -@click.option( - "--include-last/--skip-last", - default=False, - show_default=True, - help="Whether to also cancel the latest run.", -) -@click.option( - "--include-running/--skip-running", - default=True, - show_default=True, - help="Whether to also cancel running workflows.", -) -@click.argument("branch_or_pull", required=False) -def cancel_github_workflows( - branch_or_pull: Optional[str], - repo: str, - event: list[str], - include_last: bool, - include_running: bool, -) -> None: - """Cancel running or queued GitHub workflows by branch or pull request ID""" - if not github_token: - raise ClickException("Please provide GITHUB_TOKEN as an env variable") - - statuses = ("queued", "in_progress") if include_running else ("queued",) - events = event - pr = None - - if branch_or_pull is None: - title = "all jobs" if include_last else "all duplicate jobs" - elif branch_or_pull.isdigit(): - pr = get_pull_request(repo, pull_number=branch_or_pull) - title = f"pull request #{pr['number']} - {pr['title']}" - else: - title = f"branch [{branch_or_pull}]" - - print( - f"\nCancel {'active' if include_running else 'previous'} " - f"workflow runs for {title}\n" - ) - - if pr: - runs = get_runs( - repo, - statuses=statuses, - events=event, - branch=pr["head"]["ref"], - user=pr["user"]["login"], - ) - else: - user = None - branch = branch_or_pull - if branch and ":" in branch: - [user, branch] = branch.split(":", 2) - runs = get_runs( - repo, - branch=branch, - user=user, - statuses=statuses, - events=events, - ) - - # sort old jobs to the front, so to cancel older jobs first - runs = sorted(runs, key=lambda x: x["created_at"]) - if runs: - print( - f"Found {len(runs)} potential runs of\n" - f" status: {statuses}\n event: {events}\n" - ) - else: - print(f"No {' or '.join(statuses)} workflow runs found.\n") - return - - if not include_last: - # Keep the latest run for each workflow and cancel all others - seen = set() - dups = [] - for item in reversed(runs): - key = f'{item["event"]}_{item["head_branch"]}_{item["workflow_id"]}' - if key in seen: - dups.append(item) - else: - seen.add(key) - if not dups: - print( - "Only the latest runs are in queue. " - "Use --include-last to force cancelling them.\n" - ) - return - runs = dups[::-1] - - last_sha = None - - print(f"\nCancelling {len(runs)} jobs...\n") - for entry in runs: - head_commit = entry["head_commit"] - if head_commit["id"] != last_sha: - last_sha = head_commit["id"] - print("") - print_commit(head_commit, entry["head_branch"]) - try: - print(f"[{entry['status']}] {entry['name']}", end="\r") - cancel_run(repo, entry["id"]) - print(f"[Canceled] {entry['name']} ") - except ClickException as error: - print(f"[Error: {error.message}] {entry['name']} ") - print("") - - -if __name__ == "__main__": - # pylint: disable=no-value-for-parameter - cancel_github_workflows()