sdk automation, support breaking change detection (#40917)

* script

* set or increase version

* support data-plane

* group id

* replace

* fix codegen

* get_latest_ga should only return GA version

* format

* fix ga version get bug

eng/mgmt/automation/changelog.py

@@ -32,10 +32,10 @@ def main():
     if args.get("compile"):
         compile_arm_package(sdk_root, service)
 
-    versions = get_version(sdk_root, service).split(";")
+    versions = get_version(sdk_root, GROUP_ID, service).split(";")
     stable_version = versions[1]
     current_version = versions[2]
-    compare_with_maven_package(sdk_root, service, stable_version, current_version)
+    compare_with_maven_package(sdk_root, GROUP_ID, service, stable_version, current_version)
 
 
 if __name__ == "__main__":

eng/mgmt/automation/generate.py

@@ -13,10 +13,11 @@
 from parameters import *
 from utils import (
     set_or_increase_version,
-    set_or_default_version,
     update_service_ci_and_pom,
     update_root_pom,
     update_version,
+    get_latest_ga_version,
+    get_latest_release_version,
 )
 from generate_data import (
     sdk_automation as sdk_automation_data,
@@ -129,6 +130,7 @@ def sdk_automation_autorest(config: dict) -> List[dict]:
     api_specs_file = os.path.join(base_dir, API_SPECS_FILE)
 
     packages = []
+    breaking = False
     if "relatedReadmeMdFiles" not in config or not config["relatedReadmeMdFiles"]:
         return packages
 
@@ -178,7 +180,12 @@ def sdk_automation_autorest(config: dict) -> List[dict]:
                 tag=tag,
             )
             if succeeded:
-                compile_arm_package(sdk_root, module)
+                compile_succeeded = compile_arm_package(sdk_root, module)
+                if compile_succeeded:
+                    stable_version = get_latest_ga_version(GROUP_ID, module, stable_version)
+                    breaking, changelog = compare_with_maven_package(
+                        sdk_root, GROUP_ID, service, stable_version, current_version, module
+                    )
 
             packages.append(
                 {
@@ -196,6 +203,7 @@ def sdk_automation_autorest(config: dict) -> List[dict]:
                     "apiViewArtifact": next(iter(glob.glob("{0}/target/*-sources.jar".format(output_folder))), None),
                     "language": "Java",
                     "result": "succeeded" if succeeded else "failed",
+                    "changelog": {"content": changelog, "hasBreakingChange": breaking},
                 }
             )
 
@@ -236,6 +244,7 @@ def sdk_automation_typespec_project(tsp_project: str, config: dict) -> dict:
     spec_root = os.path.abspath(config["specFolder"])
     head_sha: str = config["headSha"]
     repo_url: str = config["repoHttpsUrl"]
+    breaking: bool = False
 
     succeeded, require_sdk_integration, sdk_folder, service, module = generate_typespec_project(
         tsp_project, sdk_root, spec_root, head_sha, repo_url, remove_before_regen=True, group_id=GROUP_ID
@@ -244,12 +253,25 @@ def sdk_automation_typespec_project(tsp_project: str, config: dict) -> dict:
     if succeeded:
         # TODO (weidxu): move to typespec-java
         if require_sdk_integration:
-            set_or_default_version(sdk_root, GROUP_ID, module)
             update_service_ci_and_pom(sdk_root, service, GROUP_ID, module)
             update_root_pom(sdk_root, service)
 
+        stable_version, current_version = set_or_increase_version(sdk_root, GROUP_ID, module)
+        update_parameters(None)
+        output_folder = OUTPUT_FOLDER_FORMAT.format(service)
+        update_version(sdk_root, output_folder)
+
         # compile
         succeeded = compile_arm_package(sdk_root, module)
+        if succeeded:
+            breaking, changelog = compare_with_maven_package(
+                sdk_root,
+                GROUP_ID,
+                service,
+                get_latest_ga_version(GROUP_ID, module, stable_version),
+                current_version,
+                module,
+            )
 
     # output
     if sdk_folder and module and service:
@@ -272,6 +294,7 @@ def sdk_automation_typespec_project(tsp_project: str, config: dict) -> dict:
             "apiViewArtifact": next(iter(glob.glob("{0}/target/*-sources.jar".format(sdk_folder))), None),
             "language": "Java",
             "result": result,
+            "changelog": {"content": changelog, "hasBreakingChange": breaking},
         }
     else:
         # no info about package, abort with result=failed
@@ -343,7 +366,8 @@ def main():
     if succeeded:
         succeeded = compile_arm_package(sdk_root, module)
         if succeeded:
-            compare_with_maven_package(sdk_root, service, stable_version, current_version, module)
+            latest_release_version = get_latest_release_version(stable_version, current_version)
+            compare_with_maven_package(sdk_root, GROUP_ID, service, latest_release_version, current_version, module)
 
             if args.get("auto_commit_external_change") and args.get("user_name") and args.get("user_email"):
                 pwd = os.getcwd()

eng/mgmt/automation/generate_data.py

@@ -9,6 +9,8 @@
 import subprocess
 import yaml
 import requests
+from utils import get_latest_ga_version
+from generate_utils import compare_with_maven_package
 from typing import List, Tuple, Optional
 
 from parameters import *
@@ -30,6 +32,8 @@ def sdk_automation_typespec_project(tsp_project: str, config: dict) -> dict:
     spec_root = os.path.abspath(config["specFolder"])
     head_sha: str = config["headSha"]
     repo_url: str = config["repoHttpsUrl"]
+    breaking: bool = False
+    changelog: str = ""
 
     succeeded, require_sdk_integration, sdk_folder, service, module = generate_typespec_project(
         tsp_project, sdk_root, spec_root, head_sha, repo_url
@@ -46,28 +50,45 @@ def sdk_automation_typespec_project(tsp_project: str, config: dict) -> dict:
 
     if succeeded:
         # TODO (weidxu): move to typespec-java
+        stable_version, current_version = set_or_default_version(sdk_root, GROUP_ID, module)
         if require_sdk_integration:
-            set_or_default_version(sdk_root, GROUP_ID, module)
             update_service_ci_and_pom(sdk_root, service, GROUP_ID, module)
             update_root_pom(sdk_root, service)
 
         # compile
         succeeded = compile_package(sdk_root, GROUP_ID, module)
-
-        if not succeeded:
+        if succeeded:
+            breaking, changelog = compare_with_maven_package(
+                sdk_root,
+                GROUP_ID,
+                service,
+                get_latest_ga_version(GROUP_ID, module, stable_version),
+                current_version,
+                module,
+            )
+        else:
             # check whether this is migration from Swagger
             clean_sdk_folder_succeeded = clean_sdk_folder_if_swagger(sdk_root, sdk_folder)
             if clean_sdk_folder_succeeded:
                 # re-generate
                 succeeded, require_sdk_integration, sdk_folder, service, module = generate_typespec_project(
                     tsp_project, sdk_root, spec_root, head_sha, repo_url
                 )
+                stable_version, current_version = set_or_default_version(sdk_root, GROUP_ID, module)
                 if require_sdk_integration:
-                    set_or_default_version(sdk_root, GROUP_ID, module)
                     update_service_ci_and_pom(sdk_root, service, GROUP_ID, module)
                     update_root_pom(sdk_root, service)
                 # compile
                 succeeded = compile_package(sdk_root, GROUP_ID, module)
+                if succeeded:
+                    breaking, changelog = compare_with_maven_package(
+                        sdk_root,
+                        GROUP_ID,
+                        service,
+                        get_latest_ga_version(GROUP_ID, module, stable_version),
+                        current_version,
+                        module,
+                    )
 
     # output
     if sdk_folder and module and service:
@@ -90,6 +111,7 @@ def sdk_automation_typespec_project(tsp_project: str, config: dict) -> dict:
             "apiViewArtifact": next(iter(glob.glob("{0}/target/*-sources.jar".format(sdk_folder))), None),
             "language": "Java",
             "result": result,
+            "changelog": {"content": changelog, "hasBreakingChange": breaking},
         }
     else:
         # no info about package, abort with result=failed
@@ -224,8 +246,21 @@ def sdk_automation_readme(readme_file_abspath: str, packages: List[dict], sdk_ro
 
         generated_folder = "sdk/{0}/{1}".format(service, module)
 
+        breaking = False
+        changelog = ""
+
         if succeeded:
-            compile_package(sdk_root, GROUP_ID, module)
+            stable_version, current_version = set_or_default_version(sdk_root, GROUP_ID, module)
+            succeeded = compile_package(sdk_root, GROUP_ID, module)
+            if succeeded:
+                breaking, changelog = compare_with_maven_package(
+                    sdk_root,
+                    GROUP_ID,
+                    service,
+                    get_latest_ga_version(GROUP_ID, module, stable_version),
+                    current_version,
+                    module,
+                )
 
         artifacts = ["{0}/pom.xml".format(generated_folder)]
         artifacts += [jar for jar in glob.glob("{0}/target/*.jar".format(generated_folder))]
@@ -245,6 +280,7 @@ def sdk_automation_readme(readme_file_abspath: str, packages: List[dict], sdk_ro
                 "apiViewArtifact": next(iter(glob.glob("{0}/target/*-sources.jar".format(generated_folder))), None),
                 "language": "Java",
                 "result": result,
+                "changelog": {"content": changelog, "hasBreakingChange": breaking},
             }
         )
 

eng/mgmt/automation/generate_utils.py

@@ -189,33 +189,22 @@ def update_changelog(changelog_file, changelog):
     logging.info("[Changelog][Success] Write to changelog")
 
 
-def compare_with_maven_package(sdk_root: str, service: str, stable_version: str, current_version: str, module: str):
-    if stable_version == current_version:
+def compare_with_maven_package(
+    sdk_root: str, group_id: str, service: str, previous_version: str, current_version: str, module: str
+):
+    if previous_version == current_version or previous_version is None:
         logging.info("[Changelog][Skip] no previous version")
-        return
-
-    if "-beta." in current_version and "-beta." not in stable_version:
-        # if current version is preview, try compare it with a previous preview release
-
-        version_pattern = r"\d+\.\d+\.\d+-beta\.(\d+)?"
-        beta_version_int = int(re.match(version_pattern, current_version).group(1))
-        if beta_version_int > 1:
-            previous_beta_version_int = beta_version_int - 1
-            previous_beta_version = current_version.replace(
-                "-beta." + str(beta_version_int),
-                "-beta." + str(previous_beta_version_int),
-            )
-            stable_version = previous_beta_version
+        return False, ""
 
     logging.info(
-        "[Changelog] Compare stable version {0} with current version {1}".format(stable_version, current_version)
+        "[Changelog] Compare stable version {0} with current version {1}".format(previous_version, current_version)
     )
 
     r = requests.get(
         MAVEN_URL.format(
-            group_id=GROUP_ID.replace(".", "/"),
+            group_id=group_id.replace(".", "/"),
             artifact_id=module,
-            version=stable_version,
+            version=previous_version,
         )
     )
     r.raise_for_status()
@@ -237,14 +226,16 @@ def compare_with_maven_package(sdk_root: str, service: str, stable_version: str,
             logging.error("[Changelog][Skip] Cannot get changelog")
     finally:
         os.remove(old_jar)
+    return breaking, changelog
 
 
 def get_version(
     sdk_root: str,
+    group_id: str,
     module: str,
 ) -> Union[str, None]:
     version_file = os.path.join(sdk_root, "eng/versioning/version_client.txt")
-    project = "{0}:{1}".format(GROUP_ID, module)
+    project = "{0}:{1}".format(group_id, module)
 
     with open(version_file, "r") as fin:
         for line in fin.readlines():

eng/mgmt/automation/parameters.py

@@ -12,7 +12,8 @@
 OUTPUT_FOLDER_FORMAT = None
 
 # Constant parameters
-MAVEN_URL = "https://repo1.maven.org/maven2/{group_id}/{artifact_id}/{version}/{artifact_id}-{version}.jar"
+MAVEN_HOST = "https://repo1.maven.org/maven2"
+MAVEN_URL = MAVEN_HOST + "/{group_id}/{artifact_id}/{version}/{artifact_id}-{version}.jar"
 
 SDK_ROOT = "../../../"  # related to file dir
 AUTOREST_CORE_VERSION = "3.9.7"

eng/mgmt/automation/sdk_generate.py

@@ -281,7 +281,7 @@ def main():
     if succeeded:
         succeeded = compile_arm_package(sdk_root, module)
         if succeeded:
-            compare_with_maven_package(sdk_root, service, stable_version, current_version, module)
+            compare_with_maven_package(sdk_root, GROUP_ID, service, stable_version, current_version, module)
 
             if args.get("auto_commit_external_change") and args.get("user_name") and args.get("user_email"):
                 pwd = os.getcwd()

eng/mgmt/automation/utils.py

@@ -5,13 +5,16 @@
 import re
 import subprocess
 import platform
+import requests
+from functools import cmp_to_key
 from typing import Tuple
 
 from parameters import CI_HEADER
 from parameters import CI_FORMAT
 from parameters import POM_FORMAT
 from parameters import POM_MODULE_FORMAT
 from parameters import DEFAULT_VERSION
+from parameters import MAVEN_HOST
 
 
 # Add two more indent for list in yaml dump
@@ -350,3 +353,82 @@ def set_or_increase_version(
 
 def is_windows():
     return platform.system().lower() == "windows"
+
+
+def get_latest_release_version(previous_version: str, current_version: str) -> str:
+    if "-beta." in current_version and "-beta." not in previous_version:
+        # if current version is preview, try compare it with a previous preview release
+
+        version_pattern = r"\d+\.\d+\.\d+-beta\.(\d+)?"
+        beta_version_int = int(re.match(version_pattern, current_version).group(1))
+        if beta_version_int > 1:
+            previous_beta_version_int = beta_version_int - 1
+            previous_beta_version = current_version.replace(
+                "-beta." + str(beta_version_int),
+                "-beta." + str(previous_beta_version_int),
+            )
+            previous_version = previous_beta_version
+    return previous_version
+
+
+def get_latest_ga_version(group_id: str, module: str, previous_version: str) -> str:
+    """
+    If previous_version is GA, return previous_version,
+    Otherwise, return previous GA version before previous beta.
+    None if no previous GA version.
+    """
+    if "-beta." not in previous_version:
+        return previous_version
+
+    # No previous GA version
+    if previous_version.startswith("1.0.0-beta."):
+        return None
+
+    group_path = group_id.replace(".", "/")
+
+    response = requests.get(f"{MAVEN_HOST}/{group_path}/{module}")
+
+    response.raise_for_status()
+
+    ga_version_pattern = r"<a href=\"(\d+\.\d+\.\d+)\/?"
+    ga_versions = [v.group(1) for v in re.finditer(ga_version_pattern, response.text, re.S)]
+    previous_ga_versions = sorted(
+        [v for v in ga_versions if compare_version(v, previous_version) < 0],
+        key=cmp_to_key(compare_version),
+        reverse=True,
+    )
+
+    if len(previous_ga_versions) > 0:
+        return previous_ga_versions[0]
+
+    return None
+
+
+def compare_version(v1: str, v2: str) -> int:
+    """
+    If v1 < v2, return -1;
+    Else if v1 > v2, return 1;
+    Else return 0;
+    """
+    ga_version_pattern = r"(\d+)\.(\d)+\.(\d+)?"
+    v1_major_version = int(re.match(ga_version_pattern, v1).group(1))
+    v1_minor_version = int(re.match(ga_version_pattern, v1).group(2))
+    v1_patch_version = int(re.match(ga_version_pattern, v1).group(3))
+
+    v2_major_version = int(re.match(ga_version_pattern, v2).group(1))
+    v2_minor_version = int(re.match(ga_version_pattern, v2).group(2))
+    v2_patch_version = int(re.match(ga_version_pattern, v2).group(3))
+
+    if v1_major_version < v2_major_version:
+        return -1
+    if v1_major_version > v2_major_version:
+        return 1
+    if v1_minor_version < v2_minor_version:
+        return -1
+    if v1_minor_version > v2_minor_version:
+        return 1
+    if v1_patch_version < v2_patch_version:
+        return -1
+    if v1_patch_version > v2_patch_version:
+        return 1
+    return 0