Code report and ChangeLog improvment [skip ci]

azure-sdk-tools/packaging_tools/change_log.py

@@ -150,46 +150,52 @@ def build_change_log(old_report, new_report):
 
     return change_log
 
+def get_report_from_parameter(input_parameter):
+    if ":" in input_parameter:
+        package_name, version = input_parameter.split(":")
+        from .code_report import main
+        result = main(
+            package_name,
+            version=version if version not in ["pypi", "latest"] else None,
+            last_pypi=version == "pypi"
+        )
+        if not result:
+            raise ValueError("Was not able to build a report")
+        if len(result) == 1:
+            with open(result[0], "r") as fd:
+                return json.load(fd)
+
+        raise NotImplementedError("Multi-api changelog not yet implemented")
+
+    with open(input_parameter, "r") as fd:
+        return json.load(fd)
+
 
 if __name__ == "__main__":
     import argparse
 
     parser = argparse.ArgumentParser(
         description='ChangeLog computation',
-        formatter_class=argparse.RawTextHelpFormatter,
-        epilog="Package Name and Package Print are mutually exclusive and at least one must be provided"
     )
-    parser.add_argument('base_input',
-                        help='Input base fingerprint')
-    parser.add_argument('--package-name', '-n',
-                        dest='package_name',
-                        help='Package name to test. Must be importable.')
-    parser.add_argument('--package-print', '-p',
-                        dest='package_print',
-                        help='Package name to test. Must be importable.')
+    parser.add_argument('base',
+                        help='Base. Could be a file path, or <package_name>:<version>. Version can be pypi, latest or a real version')
+    parser.add_argument('latest',
+                        help='Latest. Could be a file path, or <package_name>:<version>. Version can be pypi, latest or a real version')
+
     parser.add_argument("--debug",
                         dest="debug", action="store_true",
                         help="Verbosity in DEBUG mode")
 
     args = parser.parse_args()
 
-    main_logger = logging.getLogger()
-    logging.basicConfig()
-    main_logger.setLevel(logging.DEBUG if args.debug else logging.INFO)
-
-    if args.package_name:
-        raise NotImplementedError("FIXME")
+    logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO)
 
-    if args.package_print:
-        with open(args.package_print) as fd:
-            new_report = json.load(fd)
+    old_report = get_report_from_parameter(args.base)
+    new_report = get_report_from_parameter(args.latest)
 
-    with open(args.base_input) as fd:
-        old_report = json.load(fd)
-
-    result = diff(old_report, new_report)
-    with open("result.json", "w") as fd:
-        json.dump(result, fd)
+    # result = diff(old_report, new_report)
+    # with open("result.json", "w") as fd:
+    #     json.dump(result, fd)
 
     change_log = build_change_log(old_report, new_report)
     print(change_log.build_md())

azure-sdk-tools/packaging_tools/code_report.py

@@ -4,9 +4,19 @@
 import logging
 import pkgutil
 from pathlib import Path
+import subprocess
 import types
 from typing import Dict, Any, Optional
 
+# Because I'm subprocessing myself, I need to do weird thing as import.
+try:
+    # If I'm started as a module __main__
+    from .venvtools import create_venv_with_package
+except ModuleNotFoundError:
+    # If I'm started by my main directly
+    from venvtools import create_venv_with_package
+
+
 _LOGGER = logging.getLogger(__name__)
 
 def parse_input(input_parameter):
@@ -104,26 +114,64 @@ def create_report_from_func(function_attr):
         })
     return func_content
 
-def main(input_parameter: str, output_filename: Optional[str] = None, version: Optional[str] = None):
+def main(input_parameter: str, version: Optional[str] = None, no_venv: bool = False, pypi: bool = False, last_pypi: bool = False):
     package_name, module_name = parse_input(input_parameter)
-    report = create_report(module_name)
 
-    version = version or "latest"
+    if (version or pypi or last_pypi) and not no_venv:
+        if version:
+            versions = [version]
+        else:
+            _LOGGER.info(f"Download versions of {package_name} on PyPI")
+            from .pypi import PyPIClient
+            client = PyPIClient()
+            versions = [str(v) for v in client.get_ordered_versions(package_name)]
+            _LOGGER.info(f"Got {versions}")
+            if last_pypi:
+                _LOGGER.info(f"Only keep last PyPI version")
+                versions = [versions[-1]]
+
+        for version in versions:
+            _LOGGER.info(f"Installing version {version} of {package_name} in a venv")
+            with create_venv_with_package([f"{package_name}=={version}"]) as venv:
+                args = [
+                    venv.env_exe,
+                    __file__,
+                    "--no-venv",
+                    "--version",
+                    version,
+                    input_parameter
+                ]
+                try:
+                    subprocess.check_call(args)
+                except subprocess.CalledProcessError:
+                    # If it fail, just assume this version is too old to get an Autorest report
+                    _LOGGER.warning(f"Version {version} seems to be too old to build a report (probably not Autorest based)")
+        # Files have been written by the subprocess
+        return
+
+    modules = find_autorest_generated_folder(module_name)
+    result = []
+    for module_name in modules:
+        _LOGGER.info(f"Working on {module_name}")
+
+        report = create_report(module_name)
+        version = version or "latest"
 
-    if not output_filename:
-        split_package_name = input_parameter.split('#')
         output_filename = Path(package_name) / Path("code_reports") / Path(version)
-        if len(split_package_name) == 2:
-            output_filename /= Path(split_package_name[1]+".json")
+
+        module_for_path = get_sub_module_part(package_name, module_name)
+        if module_for_path:
+            output_filename /= Path(module_for_path+".json")
         else:
             output_filename /= Path("report.json")
-    else:
-        output_filename = Path(output_filename)
 
-    output_filename.parent.mkdir(parents=True, exist_ok=True)
+        output_filename.parent.mkdir(parents=True, exist_ok=True)
 
-    with open(output_filename, "w") as fd:
-        json.dump(report, fd, indent=2)
+        with open(output_filename, "w") as fd:
+            json.dump(report, fd, indent=2)
+            _LOGGER.info(f"Report written to {output_filename}")
+        result.append(output_filename)
+    return result
 
 def find_autorest_generated_folder(module_prefix="azure"):
     """Find all Autorest generated code in that module prefix.
@@ -132,60 +180,38 @@ def find_autorest_generated_folder(module_prefix="azure"):
     _LOGGER.info(f"Looking for Autorest generated package in {module_prefix}")
 
     # Manually skip some namespaces for now
-    if module_prefix in ["azure.cli", "azure.storage"]:
+    if module_prefix in ["azure.cli", "azure.storage", "azure.servicemanagement", "azure.servicebus"]:
         _LOGGER.info(f"Skip {module_prefix}")
         return []
 
     result = []
-    prefix_module = importlib.import_module(module_prefix)
-    for _, sub_package, ispkg in pkgutil.iter_modules(prefix_module.__path__, module_prefix+"."):
-        try:
-            # ASM has a "models", but is not autorest. Patch it widly for now.
-            if sub_package in ["azure.servicemanagement", "azure.storage", "azure.servicebus"]:
-                continue
-
-            _LOGGER.debug(f"Try {sub_package}")
-            model_module = importlib.import_module(".models", sub_package)
-
-            # If not exception, we MIGHT have found it, but cannot be a file.
-            # Keep continue to try to break it, file module have no __path__
-            model_module.__path__
-            _LOGGER.info(f"Found {sub_package}")
-            result.append(sub_package)
-        except (ModuleNotFoundError, AttributeError):
-            # No model, might dig deeper
+    try:
+        _LOGGER.debug(f"Try {module_prefix}")
+        model_module = importlib.import_module(".models", module_prefix)
+        # If not exception, we MIGHT have found it, but cannot be a file.
+        # Keep continue to try to break it, file module have no __path__
+        model_module.__path__
+        _LOGGER.info(f"Found {module_prefix}")
+        result.append(module_prefix)
+    except (ModuleNotFoundError, AttributeError):
+        # No model, might dig deeper
+        prefix_module = importlib.import_module(module_prefix)
+        for _, sub_package, ispkg in pkgutil.iter_modules(prefix_module.__path__, module_prefix+"."):
             if ispkg:
                 result += find_autorest_generated_folder(sub_package)
     return result
 
-def build_them_all():
-    """Build all reports for all packages.
-    """
-    root = Path(__file__).parent.parent.parent # Root of the repo
-
-    packages = dict()
-    for module_name in find_autorest_generated_folder():
 
-        main_module = importlib.import_module(module_name)
-
-        package_name = list(Path(main_module.__path__[0]).relative_to(root).parents)[-2]
-        packages.setdefault(package_name, set()).add(module_name)
+def get_sub_module_part(package_name, module_name):
+    """Assuming package is azure-mgmt-compute and module name is azure.mgmt.compute.v2018-08-01
+    will return v2018-08-01
+    """
+    sub_module_from_package = package_name.replace("-", ".")
+    if not module_name.startswith(sub_module_from_package):
+        _LOGGER.warning(f"Submodule {module_name} does not start with package name {package_name}")
+        return
+    return module_name[len(sub_module_from_package)+1:]
 
-    for package_name, sub_module_list in packages.items():
-        _LOGGER.info(f"Processing {package_name}")
-        package_name_str = str(package_name)
-        if len(sub_module_list) == 1:
-            main(package_name_str)
-        else:
-            for sub_module in sub_module_list:
-                _LOGGER.info(f"\tProcessing sub-module {sub_module}")
-                sub_module_from_package = package_name_str.replace("-", ".")
-                if not sub_module.startswith(sub_module_from_package):
-                    _LOGGER.warning(f"Submodule {sub_module} does not package name {package_name}")
-                    continue
-                sub_module_last_part = sub_module[len(sub_module_from_package)+1:]
-                _LOGGER.info(f"Calling main with {package_name}#{sub_module_last_part}")
-                main(f"{package_name}#{sub_module_last_part}")
 
 if __name__ == "__main__":
     import argparse
@@ -196,23 +222,24 @@ def build_them_all():
     )
     parser.add_argument('package_name',
                         help='Package name.')
-    parser.add_argument('--output-file', '-o',
-                        dest='output_file',
-                        help='Output file. [default: ./<package_name>/code_reports/<version>/<module_name>.json]')
     parser.add_argument('--version', '-v',
                         dest='version',
                         help='The version of the package you want. By default, latest and current branch.')
+    parser.add_argument('--no-venv',
+                        dest='no_venv', action="store_true",
+                        help="If version is provided, this will assume the current accessible package is already this version. You should probably not use it.")
+    parser.add_argument('--pypi',
+                        dest='pypi', action="store_true",
+                        help="If provided, build report for all versions on pypi of this package.")
+    parser.add_argument('--last-pypi',
+                        dest='last_pypi', action="store_true",
+                        help="If provided, build report for last version on pypi of this package.")
     parser.add_argument("--debug",
                         dest="debug", action="store_true",
                         help="Verbosity in DEBUG mode")
 
     args = parser.parse_args()
 
-    main_logger = logging.getLogger()
-    logging.basicConfig()
-    main_logger.setLevel(logging.DEBUG if args.debug else logging.INFO)
+    logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO)
 
-    if args.package_name == "all":
-        build_them_all()
-    else:
-        main(args.package_name, args.output_file, args.version)
+    main(args.package_name, args.version, args.no_venv, args.pypi, args.last_pypi)

azure-sdk-tools/packaging_tools/venvtools.py

@@ -0,0 +1,46 @@
+from contextlib import contextmanager
+import tempfile
+import subprocess
+import venv
+
+class ExtendedEnvBuilder(venv.EnvBuilder):
+    """An extended env builder which saves the context, to have access
+    easily to bin path and such.
+    """
+
+    def __init__(self, *args, **kwargs):
+        self.context = None
+        super(ExtendedEnvBuilder, self).__init__(*args, **kwargs)
+
+    def ensure_directories(self, env_dir):
+        self.context = super(ExtendedEnvBuilder, self).ensure_directories(env_dir)
+        return self.context
+
+
+def create(env_dir, system_site_packages=False, clear=False,
+                    symlinks=False, with_pip=False, prompt=None):
+    """Create a virtual environment in a directory."""
+    builder = ExtendedEnvBuilder(system_site_packages=system_site_packages,
+                                 clear=clear, symlinks=symlinks, with_pip=with_pip,
+                                 prompt=prompt)
+    builder.create(env_dir)
+    return builder.context
+
+@contextmanager
+def create_venv_with_package(packages):
+    """Create a venv with these packages in a temp dir and yielf the env.
+
+    packages should be an iterable of pip version instructio (e.g. package~=1.2.3)
+    """
+    with tempfile.TemporaryDirectory() as tempdir:
+        myenv = create(tempdir, with_pip=True)
+        pip_call = [
+            myenv.env_exe,
+            "-m",
+            "pip",
+            "install",
+        ]
+        pip_call += packages
+        if packages:
+            subprocess.check_call(pip_call)
+        yield myenv