-
Notifications
You must be signed in to change notification settings - Fork 24
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix(libraries-release-data): add service name lookup from artifact id…
… for gapic libraries, add apiary libraries and spring cloud gcp to release data table. (#6437) Changes in this pr: - Add script to infer artifact id to service name match from StubSettings. (closing #6390 in favor of this pr) - Add script to parse artifact id and service name for apiary libraries from discovery docs. - Manually add spring-cloud-gcp to tracked library list. Note that service name column is borrowed here for tool name. (this is to match tool release dates) - revamped `fetch-library-data.sh` to require less strict structure in the returned maven page, so it can be reused for apiary lists. - Other minor cleanups. Match sample analysis: From sampled services from 2023-01-01 to now, Before this change 145/235 did not have a match, after this change 53/235 missing match. future enhancements: To infer gapic libraries artifact id and service name closer to source (from googleapis yaml/protos).
- Loading branch information
Showing
4 changed files
with
178 additions
and
28 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| #!/bin/bash | ||
|
|
||
| # This script should download | ||
| # discovery docs: git@github.com:googleapis/discovery-artifact-manager.git | ||
| # apiary repo git@github.com:googleapis/google-api-java-client-services.git | ||
| # from discovery docs for each service | ||
| # parse artifact-id ("name") and service name ("rootUrl"). | ||
|
|
||
| # Run this script from repo root dir | ||
| # input: N/A | ||
| # output: txt file with comma separated group_id, artifact_id, service_name. | ||
|
|
||
| git clone https://github.com/googleapis/discovery-artifact-manager.git | ||
|
|
||
| cd ./discovery-artifact-manager/discoveries || exit | ||
| output_filename="../../libraries-release-data/artifacts_to_services_apiary.txt" | ||
|
|
||
| # install jq to extract info from JSON data | ||
| sudo apt-get update | ||
| sudo apt-get install -q -y jq | ||
|
|
||
| # loop through dicovery json files | ||
| for file in *.json; do | ||
| # Use jq to extract the "name" field | ||
|
|
||
| # group_id logic: https://github.com/googleapis/google-api-java-client-services/blob/421c5d6ed56d5eb1257d3fc057d7d6b4fd2f9bb7/generator/src/googleapis/codegen/utilities/maven_utils.py#L50 | ||
| # artifact_id logic: https://github.com/googleapis/google-api-java-client-services/blob/421c5d6ed56d5eb1257d3fc057d7d6b4fd2f9bb7/generator/src/googleapis/codegen/utilities/maven_utils.py#L42-L47 | ||
| # default_host https://github.com/googleapis/discovery-artifact-manager/blob/9f6638a9950991d4fe67d75bdb539e6d2be20541/google-api-client-generator/src/googleapis/codegen/languages/java/default/templates/___package___/___api_className___.java.tmpl#L44 | ||
| artifact_id_suffix=$(jq -r '.name' "$file") | ||
| default_host=$(jq -r '.rootUrl' "$file") | ||
| owner_domain=$(jq -r '.ownerDomain' "$file") | ||
|
|
||
| if [[ "$default_host" =~ ^https:// ]] && [ -n "$artifact_id_suffix" ] && [ -n "$owner_domain" ]; then | ||
| if [[ "$owner_domain" != 'google.com' ]]; then | ||
| echo "$owner_domain ==============" | ||
| continue | ||
| fi | ||
| group_id="com.google.apis" | ||
| service_name=$(echo "$default_host" | cut -d'/' -f3 | cut -d'.' -f1) | ||
| artifact_id="google-api-services-${artifact_id_suffix}" | ||
| echo "${group_id},${artifact_id},${service_name}" >> "$output_filename" | ||
| else | ||
| echo "$default_host: Not a valid URL or No 'name' field found in $file" | ||
| fi | ||
|
|
||
| done | ||
|
|
||
| cd ../.. | ||
|
|
||
| rm -rf discovery-artifact-manager/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| #!/bin/bash | ||
|
|
||
| # This scripts downloads google-cloud-java repo, loop through modules with names starting "java-", | ||
| # grabs artifactId from pom.xml file within submodule name starting with "google-", and | ||
| # service name from *StubSettings.java file. | ||
|
|
||
| # Run this script from repo root dir | ||
| # input: N/A | ||
| # output: txt file with comma separated artifact_id, service_name. | ||
|
|
||
| git clone https://github.com/googleapis/google-cloud-java.git | ||
|
|
||
| cd ./google-cloud-java || exit | ||
| filename="../artifacts_to_services.txt" | ||
|
|
||
| for module in $(find . -mindepth 2 -maxdepth 2 -name pom.xml | sort | xargs dirname); do | ||
| echo "module: ${module}" | ||
| # Only modules starting with java- contain client library artifacts. | ||
| if [[ ${module} != ./java-* ]]; then | ||
| echo "not a client library, continue..." | ||
| continue | ||
| fi | ||
| # special cases, add manually later. | ||
| if [[ ${module} == ./java-dns ]] || [[ ${module} == ./java-grafeas ]] || [[ ${module} == ./java-notification ]] || [[ ${module} == ./java-alloydb-connectors ]]; then | ||
| continue | ||
| fi | ||
| cd "${module}" || exit | ||
| # Find submodule with name starting with "google-", this is to exclude proto, grpc and bom folders, | ||
| # and locate artifact id of client library | ||
| folder=$(find . -mindepth 1 -maxdepth 1 -type d -name "google-*" ! -name "*-bom" ) | ||
| echo "folder: ${folder}" | ||
| cd "${folder}" || continue | ||
| artifact_id_string=$(find . -name 'pom.xml' -print -quit | xargs grep -m 1 '<artifactId>' | cut -d '>' -f 2 | cut -d '<' -f 1) | ||
| echo "artifact_id_string: ${artifact_id_string}" | ||
| cd .. # exist from folder ${folder} | ||
|
|
||
| # Find *StubSettings file, get the first line containing '.googleapis.com:443' | ||
| # Extract service name from it | ||
| string=$(find . -name '*StubSettings.java' -print -quit | xargs grep -m 1 '.googleapis.com:443') | ||
| service_name=$(echo "${string}" | grep -o '".*"' | tr -d '"' | cut -d "." -f 1 | cut -d "-" -f 1) | ||
| echo "service name: ${service_name}" | ||
| echo "${artifact_id_string}, ${service_name}" >> "$filename" | ||
| cd .. # exit from ${module} | ||
| done | ||
|
|
||
| # add handwritten libraries manually. | ||
| { | ||
| echo "google-cloud-bigquery, bigquery" | ||
| echo "google-cloud-bigtable, bigtable" | ||
| echo "google-cloud-bigquerystorage, bigquerystorage" | ||
| echo "google-cloud-datastore, datastore" | ||
| echo "google-cloud-firestore, firestore" | ||
| echo "google-cloud-logging, logging" | ||
| echo "google-cloud-pubsub, pubsub" | ||
| echo "google-cloud-pubsublite, pubsublite" | ||
| echo "google-cloud-storage, bigstore" | ||
| echo "google-cloud-storage-control, storage" | ||
| echo "google-cloud-spanner, spanner" | ||
| echo "google-cloud-dns, dns" | ||
| } >> "./artifacts_to_services.txt" | ||
|
|
||
| cd .. | ||
| mv ./google-cloud-java/artifacts_to_services.txt ./libraries-release-data/artifacts_to_services.txt | ||
| # clean up | ||
| rm -rf google-cloud-java/ |