From f8cf69fbe6ad535bff8f88c84c16265cefe507d5 Mon Sep 17 00:00:00 2001 From: Daniel Gaspar Date: Mon, 1 Feb 2021 16:16:23 +0000 Subject: [PATCH] feat(release): add github token to changelog script --- RELEASING/README.md | 2 + RELEASING/changelog.py | 77 +++++++++++++++++------------------- requirements/base.txt | 6 +-- requirements/development.in | 1 + requirements/development.txt | 7 +++- requirements/testing.txt | 1 - 6 files changed, 47 insertions(+), 47 deletions(-) diff --git a/RELEASING/README.md b/RELEASING/README.md index 5c4786692711c..eac5e7fd5cbed 100644 --- a/RELEASING/README.md +++ b/RELEASING/README.md @@ -118,6 +118,8 @@ git push upstream $SUPERSET_GITHUB_BRANCH Next, update the `CHANGELOG.md` with all the changes that are included in the release. Make sure the branch has been pushed to `upstream` to ensure the changelog generator can pick up changes since the previous release. +Change log script requires a github token and will try to use your env var GITHUB_TOKEN. +you can also pass the token using the parameter `--access_token`. Example: ```bash diff --git a/RELEASING/changelog.py b/RELEASING/changelog.py index 07daacba8cb16..1d83aad7d3624 100644 --- a/RELEASING/changelog.py +++ b/RELEASING/changelog.py @@ -17,18 +17,22 @@ # pylint: disable=no-value-for-parameter import csv as lib_csv -import json import os import re import sys from dataclasses import dataclass -from time import sleep from typing import Any, Dict, Iterator, List, Optional, Union -from urllib import request -from urllib.error import HTTPError import click +try: + from github import BadCredentialsException, Github, PullRequest +except ModuleNotFoundError: + print("PyGithub is a required package for this script") + exit(1) + +SUPERSET_REPO = "apache/superset" + @dataclass class GitLog: @@ -60,47 +64,31 @@ class GitChangeLog: We want to map a git author to a github login, for that we call github's API """ - def __init__(self, version: str, logs: List[GitLog]) -> None: + def __init__( + self, version: str, logs: List[GitLog], access_token: Optional[str] = None + ) -> None: self._version = version self._logs = logs self._github_login_cache: Dict[str, Optional[str]] = {} self._wait = 10 + github_token = access_token or os.environ.get("GITHUB_TOKEN") + self._github = Github(github_token) + self._superset_repo = "" - def _wait_github_rate_limit(self) -> None: - """ - Waits for available rate limit slots on the github API - """ - while True: - rate_limit_payload = self._fetch_github_rate_limit() - if rate_limit_payload["rate"]["remaining"] > 1: - break - print(".", end="", flush=True) - sleep(self._wait) - print() - - @staticmethod - def _fetch_github_rate_limit() -> Dict[str, Any]: - """ - Fetches current github rate limit info - """ - with request.urlopen("https://api.github.com/rate_limit") as response: - payload = json.loads(response.read()) - return payload - - def _fetch_github_pr(self, pr_number: int) -> Dict[str, Any]: + def _fetch_github_pr(self, pr_number: int) -> PullRequest: """ Fetches a github PR info """ - payload = {} try: - self._wait_github_rate_limit() - with request.urlopen( - "https://api.github.com/repos/apache/superset/pulls/" f"{pr_number}" - ) as response: - payload = json.loads(response.read()) - except HTTPError as ex: - print(f"{ex}", flush=True) - return payload + github_repo = self._github.get_repo(SUPERSET_REPO) + except BadCredentialsException as ex: + print( + f"Bad credentials to github provided" + f" use access_token parameter or set GITHUB_TOKEN" + ) + sys.exit(1) + + return github_repo.get_pull(pr_number) def _get_github_login(self, git_log: GitLog) -> Optional[str]: """ @@ -113,7 +101,7 @@ def _get_github_login(self, git_log: GitLog) -> Optional[str]: if git_log.pr_number: pr_info = self._fetch_github_pr(git_log.pr_number) if pr_info: - github_login = pr_info["user"]["login"] + github_login = pr_info.user.login else: github_login = author_name # set cache @@ -131,7 +119,7 @@ def __repr__(self) -> str: github_login = log.author result = result + ( f"- [#{log.pr_number}]" - f"(https://github.com/apache/superset/pull/{log.pr_number}) " + f"(https://github.com/{SUPERSET_REPO}/pull/{log.pr_number}) " f"{log.message} (@{github_login})\n" ) print(f"\r {i}/{len(self._logs)}", end="", flush=True) @@ -141,7 +129,7 @@ def __iter__(self) -> Iterator[Dict[str, Any]]: for log in self._logs: yield { "pr_number": log.pr_number, - "pr_link": f"https://github.com/apache/superset/pull/" + "pr_link": f"https://github.com/{SUPERSET_REPO}/pull/" f"{log.pr_number}", "message": log.message, "time": log.time, @@ -276,13 +264,20 @@ def compare(base_parameters: BaseParameters) -> None: @click.option( "--csv", help="The csv filename to export the changelog to", ) +@click.option( + "--access_token", + help="The github access token," + " if not provided will try to fetch from GITHUB_TOKEN env var", +) @click.pass_obj -def change_log(base_parameters: BaseParameters, csv: str) -> None: +def change_log(base_parameters: BaseParameters, csv: str, access_token: str) -> None: """ Outputs a changelog (by PR) """ previous_logs = base_parameters.previous_logs current_logs = base_parameters.current_logs previous_diff_logs = previous_logs.diff(current_logs) - logs = GitChangeLog(current_logs.git_ref, previous_diff_logs[::-1]) + logs = GitChangeLog( + current_logs.git_ref, previous_diff_logs[::-1], access_token=access_token + ) if csv: with open(csv, "w") as csv_file: log_items = list(logs) diff --git a/requirements/base.txt b/requirements/base.txt index 64ae5d1f25074..8b41983563243 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -77,7 +77,7 @@ prison==0.1.3 # via flask-appbuilder py==1.9.0 # via retry pyarrow==1.0.1 # via apache-superset pycparser==2.20 # via cffi -pyjwt==1.7.1 # via flask-appbuilder, flask-jwt-extended +pyjwt==1.7.1 # via apache-superset, flask-appbuilder, flask-jwt-extended pymeeus==0.3.7 # via convertdate pyparsing==2.4.7 # via apache-superset, packaging pyrsistent==0.16.1 # via -r requirements/base.in, jsonschema @@ -92,12 +92,12 @@ redis==3.5.3 # via apache-superset retry==0.9.2 # via apache-superset selenium==3.141.0 # via apache-superset simplejson==3.17.2 # via apache-superset -six==1.15.0 # via bleach, cryptography, flask-jwt-extended, flask-talisman, holidays, isodate, jsonschema, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json +six==1.15.0 # via bleach, cryptography, flask-jwt-extended, flask-talisman, holidays, isodate, jsonschema, packaging, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json slackclient==2.5.0 # via apache-superset sqlalchemy-utils==0.36.8 # via apache-superset, flask-appbuilder sqlalchemy==1.3.20 # via alembic, apache-superset, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils sqlparse==0.3.0 # via apache-superset -typing-extensions==3.7.4.3 # via aiohttp, yarl +typing-extensions==3.7.4.3 # via aiohttp, apache-superset, yarl urllib3==1.25.11 # via selenium vine==1.3.0 # via amqp, celery webencodings==0.5.1 # via bleach diff --git a/requirements/development.in b/requirements/development.in index eae28663d0d62..5ac06dc543399 100644 --- a/requirements/development.in +++ b/requirements/development.in @@ -24,3 +24,4 @@ pyhive[hive]>=0.6.1 psycopg2-binary==2.8.5 tableschema thrift>=0.11.0,<1.0.0 +pygithub>=1.54.1,<2.0.0 diff --git a/requirements/development.txt b/requirements/development.txt index af94def47c643..f0eb4938a3889 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -1,4 +1,4 @@ -# SHA1:ae0364cae066a5cb8fb543c4f568bfcdacb6c1b7 +# SHA1:b7181d683bed10ffe4892d2f07bc94a503f46b44 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -11,6 +11,7 @@ boto3==1.16.10 # via tabulator botocore==1.19.10 # via boto3, s3transfer cached-property==1.5.2 # via tableschema certifi==2020.6.20 # via requests +deprecated==1.2.11 # via pygithub et-xmlfile==1.0.1 # via openpyxl flask-cors==3.0.9 # via -r requirements/development.in future==0.18.2 # via pyhive @@ -24,8 +25,9 @@ openpyxl==3.0.5 # via tabulator pillow==7.2.0 # via -r requirements/development.in psycopg2-binary==2.8.5 # via -r requirements/development.in pydruid==0.6.1 # via -r requirements/development.in +pygithub==1.54.1 # via -r requirements/development.in pyhive[hive]==0.6.3 # via -r requirements/development.in -requests==2.24.0 # via pydruid, tableschema, tabulator +requests==2.24.0 # via pydruid, pygithub, tableschema, tabulator rfc3986==1.4.0 # via tableschema s3transfer==0.3.3 # via boto3 sasl==0.2.1 # via pyhive, thrift-sasl @@ -34,6 +36,7 @@ tabulator==1.52.5 # via tableschema thrift-sasl==0.4.2 # via pyhive thrift==0.13.0 # via -r requirements/development.in, pyhive, thrift-sasl unicodecsv==0.14.1 # via tableschema, tabulator +wrapt==1.12.1 # via deprecated xlrd==1.2.0 # via tabulator # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/testing.txt b/requirements/testing.txt index 2c97c83247ae3..8ea641b1dfa48 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -40,7 +40,6 @@ traitlets==5.0.5 # via ipython typed-ast==1.4.1 # via astroid wcwidth==0.2.5 # via prompt-toolkit websocket-client==0.57.0 # via docker -wrapt==1.12.1 # via astroid # The following packages are considered to be unsafe in a requirements file: # pip