From 5b87ef644bea5b03f202932ab3e6e11605b695d6 Mon Sep 17 00:00:00 2001 From: Laurent Mazuel Date: Fri, 14 Sep 2018 17:06:43 -0700 Subject: [PATCH] Code report and ChangeLog improvment [skip ci] --- azure-sdk-tools/packaging_tools/change_log.py | 56 +++--- .../packaging_tools/code_report.py | 159 ++++++++++-------- azure-sdk-tools/packaging_tools/venvtools.py | 46 +++++ 3 files changed, 170 insertions(+), 91 deletions(-) create mode 100644 azure-sdk-tools/packaging_tools/venvtools.py diff --git a/azure-sdk-tools/packaging_tools/change_log.py b/azure-sdk-tools/packaging_tools/change_log.py index 42dabcf50939..7518ffe496b5 100644 --- a/azure-sdk-tools/packaging_tools/change_log.py +++ b/azure-sdk-tools/packaging_tools/change_log.py @@ -150,46 +150,52 @@ def build_change_log(old_report, new_report): return change_log +def get_report_from_parameter(input_parameter): + if ":" in input_parameter: + package_name, version = input_parameter.split(":") + from .code_report import main + result = main( + package_name, + version=version if version not in ["pypi", "latest"] else None, + last_pypi=version == "pypi" + ) + if not result: + raise ValueError("Was not able to build a report") + if len(result) == 1: + with open(result[0], "r") as fd: + return json.load(fd) + + raise NotImplementedError("Multi-api changelog not yet implemented") + + with open(input_parameter, "r") as fd: + return json.load(fd) + if __name__ == "__main__": import argparse parser = argparse.ArgumentParser( description='ChangeLog computation', - formatter_class=argparse.RawTextHelpFormatter, - epilog="Package Name and Package Print are mutually exclusive and at least one must be provided" ) - parser.add_argument('base_input', - help='Input base fingerprint') - parser.add_argument('--package-name', '-n', - dest='package_name', - help='Package name to test. Must be importable.') - parser.add_argument('--package-print', '-p', - dest='package_print', - help='Package name to test. Must be importable.') + parser.add_argument('base', + help='Base. Could be a file path, or :. Version can be pypi, latest or a real version') + parser.add_argument('latest', + help='Latest. Could be a file path, or :. Version can be pypi, latest or a real version') + parser.add_argument("--debug", dest="debug", action="store_true", help="Verbosity in DEBUG mode") args = parser.parse_args() - main_logger = logging.getLogger() - logging.basicConfig() - main_logger.setLevel(logging.DEBUG if args.debug else logging.INFO) - - if args.package_name: - raise NotImplementedError("FIXME") + logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO) - if args.package_print: - with open(args.package_print) as fd: - new_report = json.load(fd) + old_report = get_report_from_parameter(args.base) + new_report = get_report_from_parameter(args.latest) - with open(args.base_input) as fd: - old_report = json.load(fd) - - result = diff(old_report, new_report) - with open("result.json", "w") as fd: - json.dump(result, fd) + # result = diff(old_report, new_report) + # with open("result.json", "w") as fd: + # json.dump(result, fd) change_log = build_change_log(old_report, new_report) print(change_log.build_md()) diff --git a/azure-sdk-tools/packaging_tools/code_report.py b/azure-sdk-tools/packaging_tools/code_report.py index a604e63061cd..1bd416d5fd3c 100644 --- a/azure-sdk-tools/packaging_tools/code_report.py +++ b/azure-sdk-tools/packaging_tools/code_report.py @@ -4,9 +4,19 @@ import logging import pkgutil from pathlib import Path +import subprocess import types from typing import Dict, Any, Optional +# Because I'm subprocessing myself, I need to do weird thing as import. +try: + # If I'm started as a module __main__ + from .venvtools import create_venv_with_package +except ModuleNotFoundError: + # If I'm started by my main directly + from venvtools import create_venv_with_package + + _LOGGER = logging.getLogger(__name__) def parse_input(input_parameter): @@ -104,26 +114,64 @@ def create_report_from_func(function_attr): }) return func_content -def main(input_parameter: str, output_filename: Optional[str] = None, version: Optional[str] = None): +def main(input_parameter: str, version: Optional[str] = None, no_venv: bool = False, pypi: bool = False, last_pypi: bool = False): package_name, module_name = parse_input(input_parameter) - report = create_report(module_name) - version = version or "latest" + if (version or pypi or last_pypi) and not no_venv: + if version: + versions = [version] + else: + _LOGGER.info(f"Download versions of {package_name} on PyPI") + from .pypi import PyPIClient + client = PyPIClient() + versions = [str(v) for v in client.get_ordered_versions(package_name)] + _LOGGER.info(f"Got {versions}") + if last_pypi: + _LOGGER.info(f"Only keep last PyPI version") + versions = [versions[-1]] + + for version in versions: + _LOGGER.info(f"Installing version {version} of {package_name} in a venv") + with create_venv_with_package([f"{package_name}=={version}"]) as venv: + args = [ + venv.env_exe, + __file__, + "--no-venv", + "--version", + version, + input_parameter + ] + try: + subprocess.check_call(args) + except subprocess.CalledProcessError: + # If it fail, just assume this version is too old to get an Autorest report + _LOGGER.warning(f"Version {version} seems to be too old to build a report (probably not Autorest based)") + # Files have been written by the subprocess + return + + modules = find_autorest_generated_folder(module_name) + result = [] + for module_name in modules: + _LOGGER.info(f"Working on {module_name}") + + report = create_report(module_name) + version = version or "latest" - if not output_filename: - split_package_name = input_parameter.split('#') output_filename = Path(package_name) / Path("code_reports") / Path(version) - if len(split_package_name) == 2: - output_filename /= Path(split_package_name[1]+".json") + + module_for_path = get_sub_module_part(package_name, module_name) + if module_for_path: + output_filename /= Path(module_for_path+".json") else: output_filename /= Path("report.json") - else: - output_filename = Path(output_filename) - output_filename.parent.mkdir(parents=True, exist_ok=True) + output_filename.parent.mkdir(parents=True, exist_ok=True) - with open(output_filename, "w") as fd: - json.dump(report, fd, indent=2) + with open(output_filename, "w") as fd: + json.dump(report, fd, indent=2) + _LOGGER.info(f"Report written to {output_filename}") + result.append(output_filename) + return result def find_autorest_generated_folder(module_prefix="azure"): """Find all Autorest generated code in that module prefix. @@ -132,60 +180,38 @@ def find_autorest_generated_folder(module_prefix="azure"): _LOGGER.info(f"Looking for Autorest generated package in {module_prefix}") # Manually skip some namespaces for now - if module_prefix in ["azure.cli", "azure.storage"]: + if module_prefix in ["azure.cli", "azure.storage", "azure.servicemanagement", "azure.servicebus"]: _LOGGER.info(f"Skip {module_prefix}") return [] result = [] - prefix_module = importlib.import_module(module_prefix) - for _, sub_package, ispkg in pkgutil.iter_modules(prefix_module.__path__, module_prefix+"."): - try: - # ASM has a "models", but is not autorest. Patch it widly for now. - if sub_package in ["azure.servicemanagement", "azure.storage", "azure.servicebus"]: - continue - - _LOGGER.debug(f"Try {sub_package}") - model_module = importlib.import_module(".models", sub_package) - - # If not exception, we MIGHT have found it, but cannot be a file. - # Keep continue to try to break it, file module have no __path__ - model_module.__path__ - _LOGGER.info(f"Found {sub_package}") - result.append(sub_package) - except (ModuleNotFoundError, AttributeError): - # No model, might dig deeper + try: + _LOGGER.debug(f"Try {module_prefix}") + model_module = importlib.import_module(".models", module_prefix) + # If not exception, we MIGHT have found it, but cannot be a file. + # Keep continue to try to break it, file module have no __path__ + model_module.__path__ + _LOGGER.info(f"Found {module_prefix}") + result.append(module_prefix) + except (ModuleNotFoundError, AttributeError): + # No model, might dig deeper + prefix_module = importlib.import_module(module_prefix) + for _, sub_package, ispkg in pkgutil.iter_modules(prefix_module.__path__, module_prefix+"."): if ispkg: result += find_autorest_generated_folder(sub_package) return result -def build_them_all(): - """Build all reports for all packages. - """ - root = Path(__file__).parent.parent.parent # Root of the repo - - packages = dict() - for module_name in find_autorest_generated_folder(): - main_module = importlib.import_module(module_name) - - package_name = list(Path(main_module.__path__[0]).relative_to(root).parents)[-2] - packages.setdefault(package_name, set()).add(module_name) +def get_sub_module_part(package_name, module_name): + """Assuming package is azure-mgmt-compute and module name is azure.mgmt.compute.v2018-08-01 + will return v2018-08-01 + """ + sub_module_from_package = package_name.replace("-", ".") + if not module_name.startswith(sub_module_from_package): + _LOGGER.warning(f"Submodule {module_name} does not start with package name {package_name}") + return + return module_name[len(sub_module_from_package)+1:] - for package_name, sub_module_list in packages.items(): - _LOGGER.info(f"Processing {package_name}") - package_name_str = str(package_name) - if len(sub_module_list) == 1: - main(package_name_str) - else: - for sub_module in sub_module_list: - _LOGGER.info(f"\tProcessing sub-module {sub_module}") - sub_module_from_package = package_name_str.replace("-", ".") - if not sub_module.startswith(sub_module_from_package): - _LOGGER.warning(f"Submodule {sub_module} does not package name {package_name}") - continue - sub_module_last_part = sub_module[len(sub_module_from_package)+1:] - _LOGGER.info(f"Calling main with {package_name}#{sub_module_last_part}") - main(f"{package_name}#{sub_module_last_part}") if __name__ == "__main__": import argparse @@ -196,23 +222,24 @@ def build_them_all(): ) parser.add_argument('package_name', help='Package name.') - parser.add_argument('--output-file', '-o', - dest='output_file', - help='Output file. [default: .//code_reports//.json]') parser.add_argument('--version', '-v', dest='version', help='The version of the package you want. By default, latest and current branch.') + parser.add_argument('--no-venv', + dest='no_venv', action="store_true", + help="If version is provided, this will assume the current accessible package is already this version. You should probably not use it.") + parser.add_argument('--pypi', + dest='pypi', action="store_true", + help="If provided, build report for all versions on pypi of this package.") + parser.add_argument('--last-pypi', + dest='last_pypi', action="store_true", + help="If provided, build report for last version on pypi of this package.") parser.add_argument("--debug", dest="debug", action="store_true", help="Verbosity in DEBUG mode") args = parser.parse_args() - main_logger = logging.getLogger() - logging.basicConfig() - main_logger.setLevel(logging.DEBUG if args.debug else logging.INFO) + logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO) - if args.package_name == "all": - build_them_all() - else: - main(args.package_name, args.output_file, args.version) + main(args.package_name, args.version, args.no_venv, args.pypi, args.last_pypi) diff --git a/azure-sdk-tools/packaging_tools/venvtools.py b/azure-sdk-tools/packaging_tools/venvtools.py new file mode 100644 index 000000000000..bb743a1b7ece --- /dev/null +++ b/azure-sdk-tools/packaging_tools/venvtools.py @@ -0,0 +1,46 @@ +from contextlib import contextmanager +import tempfile +import subprocess +import venv + +class ExtendedEnvBuilder(venv.EnvBuilder): + """An extended env builder which saves the context, to have access + easily to bin path and such. + """ + + def __init__(self, *args, **kwargs): + self.context = None + super(ExtendedEnvBuilder, self).__init__(*args, **kwargs) + + def ensure_directories(self, env_dir): + self.context = super(ExtendedEnvBuilder, self).ensure_directories(env_dir) + return self.context + + +def create(env_dir, system_site_packages=False, clear=False, + symlinks=False, with_pip=False, prompt=None): + """Create a virtual environment in a directory.""" + builder = ExtendedEnvBuilder(system_site_packages=system_site_packages, + clear=clear, symlinks=symlinks, with_pip=with_pip, + prompt=prompt) + builder.create(env_dir) + return builder.context + +@contextmanager +def create_venv_with_package(packages): + """Create a venv with these packages in a temp dir and yielf the env. + + packages should be an iterable of pip version instructio (e.g. package~=1.2.3) + """ + with tempfile.TemporaryDirectory() as tempdir: + myenv = create(tempdir, with_pip=True) + pip_call = [ + myenv.env_exe, + "-m", + "pip", + "install", + ] + pip_call += packages + if packages: + subprocess.check_call(pip_call) + yield myenv \ No newline at end of file