From 762413f2d2fd64666c17f1be2e62363c942d3551 Mon Sep 17 00:00:00 2001 From: Lewis Headden Date: Tue, 31 Oct 2017 14:59:19 -0400 Subject: [PATCH 1/5] Add support for preprocessing variants based on language --- openapi/client-generator.sh | 1 + openapi/generate_client_in_container.sh | 3 +- openapi/preprocess_spec.py | 52 +++++++++++++++---------- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/openapi/client-generator.sh b/openapi/client-generator.sh index 281665f..7768231 100755 --- a/openapi/client-generator.sh +++ b/openapi/client-generator.sh @@ -66,6 +66,7 @@ kubeclient::generator::generate_client() { -e CLEANUP_DIRS="${CLEANUP_DIRS_STRING}" \ -e KUBERNETES_BRANCH="${KUBERNETES_BRANCH}" \ -e CLIENT_VERSION="${CLIENT_VERSION}" \ + -e CLIENT_LANGUAGE="${CLIENT_LANGUAGE}" \ -e PACKAGE_NAME="${PACKAGE_NAME}" \ -e SWAGGER_CODEGEN_COMMIT="${SWAGGER_CODEGEN_COMMIT}" \ -v "${output_dir}:/output_dir" \ diff --git a/openapi/generate_client_in_container.sh b/openapi/generate_client_in_container.sh index 11a0034..52e1149 100755 --- a/openapi/generate_client_in_container.sh +++ b/openapi/generate_client_in_container.sh @@ -33,6 +33,7 @@ set -o pipefail : "${CLEANUP_DIRS?Must set CLEANUP_DIRS env var}" : "${KUBERNETES_BRANCH?Must set KUBERNETES_BRANCH env var}" : "${CLIENT_VERSION?Must set CLIENT_VERSION env var}" +: "${CLIENT_LANGUAGE?Must set CLIENT_LANGUAGE env var}" : "${PACKAGE_NAME?Must set PACKAGE_NAME env var}" : "${SWAGGER_CODEGEN_COMMIT?Must set SWAGGER_CODEGEN_COMMIT env var}" @@ -78,7 +79,7 @@ popd mkdir -p "${output_dir}" echo "--- Downloading and pre-processing OpenAPI spec" -python "${SCRIPT_ROOT}/preprocess_spec.py" "${KUBERNETES_BRANCH}" "${output_dir}/swagger.json" +python "${SCRIPT_ROOT}/preprocess_spec.py" "${CLIENT_LANGUAGE}" "${KUBERNETES_BRANCH}" "${output_dir}/swagger.json" echo "--- Cleaning up previously generated folders" for i in ${CLEANUP_DIRS}; do diff --git a/openapi/preprocess_spec.py b/openapi/preprocess_spec.py index 1ac6e53..4519e27 100644 --- a/openapi/preprocess_spec.py +++ b/openapi/preprocess_spec.py @@ -110,7 +110,7 @@ def add_custom_objects_spec(spec): return spec -def process_swagger(spec): +def process_swagger(spec, client_language): spec = add_custom_objects_spec(spec) apply_func_to_spec_operations(spec, strip_tags_from_operation_id) @@ -127,10 +127,15 @@ def process_swagger(spec): remove_model_prefixes(spec) - inline_primitive_models(spec) + inline_primitive_models(spec, preserved_primitives_for_language(client_language)) return spec +def preserved_primitives_for_language(client_language): + if client_language == "java": + return ["intstr.IntOrString"] + else: + return [] def rename_model(spec, old_name, new_name): if new_name in spec['definitions']: @@ -180,7 +185,7 @@ def remove_deprecated_models(spec): models = {} for k, v in spec['definitions'].items(): if is_model_deprecated(v): - print("Removing deprecated model %s" %k) + print("Removing deprecated model %s" % k) else: models[k] = v spec['definitions'] = models @@ -252,30 +257,38 @@ def find_replace_ref_recursive(root, ref_name, replace_map): find_replace_ref_recursive(v, ref_name, replace_map) -def inline_primitive_models(spec): +def inline_primitive_models(spec, excluded_primitives): to_remove_models = [] for k, v in spec['definitions'].items(): - if "properties" not in v: - if k == "intstr.IntOrString": - v["type"] = "object" - if "type" not in v: - v["type"] = "object" - print("Making model `%s` inline as %s..." % (k, v["type"])) - find_replace_ref_recursive(spec, "#/definitions/" + k, v) - to_remove_models.append(k) + if k not in excluded_primitives: + if "properties" not in v: + if k == "intstr.IntOrString": + v["type"] = "object" + if "type" not in v: + v["type"] = "object" + print("Making model `%s` inline as %s..." % (k, v["type"])) + find_replace_ref_recursive(spec, "#/definitions/" + k, v) + to_remove_models.append(k) for k in to_remove_models: del spec['definitions'][k] +def write_json(filename, object): + with open(filename, 'w') as out: + json.dump(object, out, sort_keys=False, indent=2, + separators=(',', ': '), ensure_ascii=True) + + def main(): - if len(sys.argv) != 3: - print("Usage:\n\n\tpython preprocess_spec.py kuberneres_branch " \ + if len(sys.argv) != 4: + print("Usage:\n\n\tpython preprocess_spec.py client_language kuberneres_branch " \ "output_spec_path") return 1 + client_language = sys.argv[1] spec_url = 'https://raw.githubusercontent.com/kubernetes/kubernetes/' \ - '%s/api/openapi-spec/swagger.json' % sys.argv[1] - output_path = sys.argv[2] + '%s/api/openapi-spec/swagger.json' % sys.argv[2] + output_path = sys.argv[3] pool = urllib3.PoolManager() with pool.request('GET', spec_url, preload_content=False) as response: @@ -283,10 +296,9 @@ def main(): print("Error downloading spec file. Reason: %s" % response.reason) return 1 in_spec = json.load(response, object_pairs_hook=OrderedDict) - out_spec = process_swagger(in_spec) - with open(output_path, 'w') as out: - json.dump(out_spec, out, sort_keys=False, indent=2, - separators=(',', ': '), ensure_ascii=True) + write_json(output_path + ".unprocessed", in_spec) + out_spec = process_swagger(in_spec, client_language) + write_json(output_path, out_spec) return 0 From 1a4a4ff8aca9796fc7099e5db71daa8ebf28c1b9 Mon Sep 17 00:00:00 2001 From: Lewis Headden Date: Tue, 31 Oct 2017 15:19:21 -0400 Subject: [PATCH 2/5] Add IntOrString mapping for Java --- openapi/java.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openapi/java.xml b/openapi/java.xml index ef04f08..ac590fd 100644 --- a/openapi/java.xml +++ b/openapi/java.xml @@ -45,6 +45,8 @@ joda false okhttp-gson + intstr.IntOrString=IntOrString + IntOrString=io.kubernetes.client.custom.IntOrString ${generator.output.path} From 0f9ff8b30afdfc78a449caddb213bbcc7b319470 Mon Sep 17 00:00:00 2001 From: Lewis Headden Date: Thu, 2 Nov 2017 10:59:11 -0400 Subject: [PATCH 3/5] Continue loop on excluded primitive --- openapi/preprocess_spec.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/openapi/preprocess_spec.py b/openapi/preprocess_spec.py index 4519e27..df28e3e 100644 --- a/openapi/preprocess_spec.py +++ b/openapi/preprocess_spec.py @@ -260,15 +260,16 @@ def find_replace_ref_recursive(root, ref_name, replace_map): def inline_primitive_models(spec, excluded_primitives): to_remove_models = [] for k, v in spec['definitions'].items(): - if k not in excluded_primitives: - if "properties" not in v: - if k == "intstr.IntOrString": - v["type"] = "object" - if "type" not in v: - v["type"] = "object" - print("Making model `%s` inline as %s..." % (k, v["type"])) - find_replace_ref_recursive(spec, "#/definitions/" + k, v) - to_remove_models.append(k) + if k in excluded_primitives: + continue + if "properties" not in v: + if k == "intstr.IntOrString": + v["type"] = "object" + if "type" not in v: + v["type"] = "object" + print("Making model `%s` inline as %s..." % (k, v["type"])) + find_replace_ref_recursive(spec, "#/definitions/" + k, v) + to_remove_models.append(k) for k in to_remove_models: del spec['definitions'][k] From ec6775b6ea09acf03b6c0e2ea1cd9e383fbe9541 Mon Sep 17 00:00:00 2001 From: Lewis Headden Date: Thu, 2 Nov 2017 11:01:11 -0400 Subject: [PATCH 4/5] Remove line break --- openapi/preprocess_spec.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openapi/preprocess_spec.py b/openapi/preprocess_spec.py index df28e3e..4ea83fe 100644 --- a/openapi/preprocess_spec.py +++ b/openapi/preprocess_spec.py @@ -276,8 +276,7 @@ def inline_primitive_models(spec, excluded_primitives): def write_json(filename, object): with open(filename, 'w') as out: - json.dump(object, out, sort_keys=False, indent=2, - separators=(',', ': '), ensure_ascii=True) + json.dump(object, out, sort_keys=False, indent=2, separators=(',', ': '), ensure_ascii=True) From 4adc3f0211ecb7096d68dec746d0f43bc4d01393 Mon Sep 17 00:00:00 2001 From: Lewis Headden Date: Thu, 2 Nov 2017 11:02:07 -0400 Subject: [PATCH 5/5] Fix typo --- openapi/preprocess_spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openapi/preprocess_spec.py b/openapi/preprocess_spec.py index 4ea83fe..19ade64 100644 --- a/openapi/preprocess_spec.py +++ b/openapi/preprocess_spec.py @@ -282,7 +282,7 @@ def write_json(filename, object): def main(): if len(sys.argv) != 4: - print("Usage:\n\n\tpython preprocess_spec.py client_language kuberneres_branch " \ + print("Usage:\n\n\tpython preprocess_spec.py client_language kubernetes_branch " \ "output_spec_path") return 1 client_language = sys.argv[1]