diff --git a/repro/common/helpers.py b/repro/common/helpers.py new file mode 100644 index 0000000..05557f8 --- /dev/null +++ b/repro/common/helpers.py @@ -0,0 +1,254 @@ +from http.client import NOT_IMPLEMENTED +import os +import platform +import requests +from requests.exceptions import HTTPError +import yaml +from rich.console import Console + +API_TIMEOUT = 300 +console = Console() + + +def get_profile_url(profile): + return get_profile_data(profile)['server_url'] + + +def get_profile_api_key(profile): + return get_profile_data(profile)['api_key'] + + +def get_profile_data(profile): + cli_config_filename = 'cli.yml' + + match platform.system().lower(): + case 'darwin': + home_dir = os.environ.get('HOME') + path = ( + f'{home_dir}/Library/Application Support/com.cloudtruth.CloudTruth-CLI/' + + f'{cli_config_filename}' + ) + case 'linux': + home_dir = os.environ.get('XDG_CONFIG_HOME') + path = f'{home_dir}/cloudtruth/{cli_config_filename}' + case 'windows': + app_data = os.environ.get('APPDATA') + path = fr'{app_data}\CloudTruth\CloudTruth CLI\config\{cli_config_filename}' + case _: + exit('Cannot determine config file path, exiting.') + + with open(path, 'r', encoding='utf_8') as file: + config = yaml.safe_load(file) + + if profile not in config['profiles']: + exit('Profile not found, check spelling or create a new profile via the CloudTruth CLI') + + return config['profiles'][profile] + + +def make_request(uri, http_method, headers, body=None): + try: + response = requests.request( + method=http_method.upper(), + url=uri, + headers=headers, + timeout=API_TIMEOUT, + json=body + ) + + response.raise_for_status() + except HTTPError as err: + console.log(err) + console.log(locals()) # TODO: Debugging-only!!! + exit() + + return response + + +def get_objects_list(cloudtruth_type, api_url, headers): + url = f'{api_url}/{cloudtruth_type}/' + return make_request(url, 'get', headers).json().get('results') + + +def get_object_by_name(name, cloudtruth_type, api_url, headers): + objs = get_objects_list(cloudtruth_type, api_url, headers) + + for obj in objs: + if obj['name'] == name: + return obj + + return None + + +def delete_object(obj, cloudtruth_type, api_url, headers): + obj_id = obj.get('id') + url = f'{api_url}/{cloudtruth_type}/{obj_id}/' + + make_request(url, 'delete', headers) + + +def get_object_by_id(object_id, cloudtruth_type, api_url, headers): + return NOT_IMPLEMENTED + + +# PROJECTS + +def create_project(name, api_url, headers, parent=None): + project_url = f'{api_url}/projects/' + depends_on = parent.get('url') if parent is not None else '' + body = { + "name": name, + "depends_on": depends_on + } + + project = make_request(project_url, 'post', headers, body).json() + + return project + + +def delete_project(project, api_url, headers): + project_and_dependents = get_project_tree_projects(project, api_url, headers) + + for project in reversed(project_and_dependents): + console.log(f"Deleting project: {project.get('name')}") + delete_object(project, 'projects', api_url, headers) + + +def project_has_dependents(project): + child_project_urls = project.get('dependents') + if len(child_project_urls) > 0: + return True + + return False + + +def project_has_parent(project): + parent_project_url = project.get('depends_on') + if parent_project_url: + return True + + return False + + +def get_project_tree_projects(project, api_url, headers, projects=None): + if project is None: + return None + if projects is None: + projects = [] + console.log(f"Adding parent, {project.get('name')} to the array of projects") + projects.append(project) # put the parent in the list on init + if project_has_dependents(project): + child_project_urls = project.get('dependents') + for child_project_url in child_project_urls: + child_project = make_request(child_project_url, 'get', headers).json() + if project_has_dependents(child_project): # if the child has dependents + console.log(f"Adding a child to the list, {child_project.get('name')} with dependencies") + projects.append(child_project) + get_project_tree_projects(child_project, api_url, headers, projects) + else: + console.log(f"Adding a child to the list, {child_project.get('name')} with no dependencies") + projects.append(child_project) + + return projects + + +def get_all_top_level_projects(api_url, headers): + all_projects = get_objects_list('projects', api_url, headers) + top_level_projects = [] + + for project in all_projects: + if project_has_parent(project) is False: + top_level_projects.append(project) + + return top_level_projects + +# ENVIRONMENTS + +def create_environment(name, api_url, headers, parent=None): + env_url = f'{api_url}/environments/' + parent_uri = parent.get('url') if parent is not None else '' + body = { + "name": name, + "parent": parent_uri + } + + env = make_request(env_url, 'post', headers, body).json() + + return env + +def delete_environment(environment, api_url, headers): + if environment == 'default': + return False + + environment_and_dependents = get_environment_tree_environments(environment, api_url, headers) + + for environment in reversed(environment_and_dependents): + console.log(f"Deleting environment: {environment.get('name')}") + delete_object(environment, 'environments', api_url, headers) + +def environment_has_dependents(environment): + child_environment_urls = environment.get('children') + if len(child_environment_urls) > 0: + return True + + return False + +def get_environment_tree_environments(environment, api_url, headers, environments=None): + if environment is None: + return None + if environments is None: + environments = [] + console.log(f"Adding parent: {environment.get('name')} to the array of environments") + environments.append(environment) + if environment_has_dependents(environment): + child_environment_urls = environment.get('children') + for child_environment_url in child_environment_urls: + child_environment = make_request(child_environment_url, 'get', headers).json() + if environment_has_dependents(child_environment): + console.log(f"Adding a child to the list, {child_environment.get('name')} with dependencies") + environments.append(child_environment) + get_environment_tree_environments(child_environment, api_url, headers, environments) + else: + console.log(f"Adding a child to the list, {child_environment.get('name')} with no dependencies") + environments.append(child_environment) + + return environments + + +# PARAMETERS + +def create_parameter(name, project, api_url, headers): + # TODO: handle secrets? + + project_id = project.get('id') + param_url = f"{api_url}/projects/{project_id}/parameters/" + body = { + "name": name + } + + param = make_request(param_url, 'post', headers, body).json() + + return param + + +def delete_project_parameters(project, api_url, headers): + project_id = project.get('id') + project_parameters_url = f"{api_url}/projects/{project_id}/parameters/" + parameters = make_request(project_parameters_url, 'get', headers).json().get('results', []) + + console.print(f"Deleting {parameters.count()} parameters from project {project.get('name')}") + for parameter in parameters: + delete_parameter(project_id, parameter, api_url, headers) + + +def delete_parameter(project_id, parameter, api_url, headers): + parameter_id = parameter.get('id') + parameter_url = f"{api_url}/projects/{project_id}/parameters/{parameter_id}" + + make_request(parameter_url, 'delete', api_url, headers) + + +# MISC + +def nuke(): + return NOT_IMPLEMENTED diff --git a/repro/ct_org_repro_init.py b/repro/ct_org_repro_init.py new file mode 100755 index 0000000..9ee94db --- /dev/null +++ b/repro/ct_org_repro_init.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 + +import argparse +from argparse import RawTextHelpFormatter +from os import environ +import shortuuid +from rich.console import Console +from common import helpers + +console = Console() + +PROJECT_PREFIX = 'proj-testing_' +ENVIRONMENT_PREFIX = 'env-testing_' +PARAMETER_PREFIX = 'param-testing_' + +EXIT_ERR = 1 +EXIT_SUCCESS = 0 + + +def parse_args(): + usage_text = ( + 'Creates or deletes projects, envs, templates, and parameters based on inputs' + ) + parser = argparse.ArgumentParser(description=usage_text, formatter_class=RawTextHelpFormatter) + parser.add_argument( + '--profile', + help='The CloudTruth CLI profile to use', + required=True, + ) + parser.add_argument( + '--projects', + help='base type, project create or delete', + action='store_true', + ) + parser.add_argument( + '--environments', + help='base type, environment create or delete', + action='store_true', + ) + parser.add_argument( + '--templates', + help='base type, template create or delete', + action='store_true', + ) + parser.add_argument( + '--params', + help='base type, parameter create or delete', + action='store_true', + ) + parser.add_argument( + '--project-root-name', + nargs='?', + metavar='', + help='set a root project for the new projects', + ) + parser.add_argument( + '--environment-root-name', + nargs='?', + metavar='', + help='set a root environment for the new environments', + ) + parser.add_argument( + '--levels', + metavar='N', + help=( + 'Number of extra levels to nest (depth).\n' + + 'each level will get the same number of child projects and environments, default is 0' + ), + type=int, + default=0, + ) + parser.add_argument( + '--create', + help='Create given types if empty and not associated, requires one or more base types to be set', + action='store_true', + ) + parser.add_argument( + '--delete', + help='Delete given types if empty and not associated, requires one or more base types to be set', + action='store_true', + ) + parser.add_argument( + '--keep', + help='Will keep the created items and not clean up', + action='store_true', + ) + parser.add_argument( + '--force', + help=( + 'Used for deleting everything associated with the given type, ' + + 'requires a base type to be set' + ), + action='store_true', + ) + parser.add_argument( + '--reset-org', + help=( + 'The nuclear option. Resets org to default state. ' + + 'Will delete everything, even previously created models' + ), + action='store_true', + ) + parser.add_argument( + '--project-count', + metavar='', + help='Number of projects to create, default is 2', + type=int, + default=2, + ) + parser.add_argument( + '--environment-count', + metavar='', + help=( + 'Number of environments to create, default is 2' + ), + type=int, + default=2, + ) + parser.add_argument( + '--template-count', + metavar='', + help=( + 'Number of templates to create per project, default is 2' + ), + type=int, + default=2, + ) + parser.add_argument( + '--parameter-count', + metavar='', + help='Number of parameters to create in each project, default is 5 per project', + type=int, + default=2, + ) + + return parser.parse_args() + + +def create_projects(count, parent_project_name=None): + created_projects = [] + parent_project = None + + if parent_project_name: + parent_project = helpers.get_object_by_name( + parent_project_name, + 'projects', + _api_url, + _headers + ) + if parent_project is None: + console.log(f'No parent project named {parent_project_name}!') + + i = 0 + while i < count: + project_name = PROJECT_PREFIX + shortuuid.uuid() + project = helpers.create_project(project_name, _api_url, _headers, parent_project) + created_projects.append(project) + i += 1 + + console.log(f'Created {len(created_projects)} projects') + + +def create_envs(count, parent): + i = 0 + while i < count: + env_name = ENVIRONMENT_PREFIX + shortuuid.uuid() + env = helpers.create_environment(env_name, _api_url, _headers, parent) + _envs.append(env) + i += 1 + + +def create_params(count, project): + i = 0 + param_name = PARAMETER_PREFIX + shortuuid.uuid() + while i < count: + helpers.create_parameter(param_name, project, _api_url, _headers) + + +def delete_all_parameters(): + projects = helpers.get_objects_list('projects', _api_url, _headers) + for project in projects: + helpers.delete_project_parameters(project, _api_url, _headers) + + +def delete_all_projects(): + projects = helpers.get_all_top_level_projects(_api_url, _headers) + + if not projects: + return None + + for project in projects: + helpers.delete_project(project, _api_url, _headers) + + return True + + +def delete_single_project(project_root_name=None): + project = helpers.get_object_by_name(project_root_name, 'projects', _api_url, _headers) + + if not project: + return None + + helpers.delete_project(project, _api_url, _headers) + return True + +def delete_single_environment(environment_root_name=None): + environment = helpers.get_object_by_name(environment_root_name, 'environments', _api_url, _headers) + + if not environment: + return None + + helpers.delete_environment(environment, _api_url, _headers) + return True + +def delete_all_environments(): + environments = helpers.get_objects_list('environments', _api_url, _headers) + + if not environments: + return None + + for environment in environments: + helpers.delete_environment(environment, _api_url, _headers) + + return True + + +def main(): + args = parse_args() + profile = args.profile + + # validate options + if args.force and args.delete is None: + console.print('force is ignored as it is only used with delete operations') + + if args.create and args.delete: + console.print('Ambiguous operations, create and delete cannot be used together') + exit(EXIT_ERR) + + global _api_key + _api_key = helpers.get_profile_api_key(profile) + if not _api_key: + exit('Profile is missing the target API key, check the CloudTruth CLI config. Exiting.') + + global _api_url + _api_url = f'{helpers.get_profile_url(profile)}/api/v1' + if not _api_url: + exit('Profile is missing the target API url, check the CloudTruth CLI config. Exiting.') + + global _headers + _headers = {'Authorization': f'Api-Key {_api_key}'} + + global _projects + _projects = [] + + global _envs + _envs = [] + + global _template_uris + _template_uris = [] + + # create projects + if args.projects and args.create: + console.print('creating projects') + project_count = args.project_count + project_root_name = args.project_root_name + create_projects(project_count, project_root_name) + if args.levels: + projects = helpers.get_objects_list('projects', _api_url, _headers) + i = 0 + while i < args.levels: + for project in projects: + create_projects(project_count, project.get('name')) + i += 1 + + # delete projects + if args.projects and args.delete: + if args.project_root_name: + console.print(f"Deleting project: {args.project_root_name} and all dependents") + result = delete_single_project(args.project_root_name) + if not result: + console.print('Project not found or could not be deleted') + exit(EXIT_ERR) + else: + console.print('Deleting all projects') + result = delete_all_projects() + if not result: + console.print('No projects found') + + # create environments + if args.environments and args.create: + console.print('creating environments') + env_count = args.environment_count + env_root_name = args.environment_root_name + create_envs(env_count, env_root_name) + if args.levels: + environments = helpers.get_objects_list('environments', _api_url, _headers) + i = 0 + while i < args.levels: + for env in environments: + create_envs(env_count, env.get('name')) + i += 1 + + # delete environments + if args.environments and args.delete: + if args.environment_root_name == 'default': + console.print('Cannot delete thedefault environment') + exit(EXIT_ERR) + if args.environment_root_name: + console.print(f"Deleting environment: {args.environment_root_name} and all dependents") + result = delete_single_environment(args.environment_root_name) + if not result: + console.print('Environment not found or could not be deleted') + exit(EXIT_ERR) + else: + console.print('Deleting all environments (except default)') + result = delete_all_environments() + if not result: + console.print('No environments found') + + # create parameters + if args.params and args.create: + param_count = args.params + if param_count > 0 and args.delete_force is not True: + console.print('Creating parameters') + for project in _projects: + create_params(param_count, project) + console.print(f'Created {param_count} parameters in each of {_projects.count()} projects' ) + + # if args.delete_project_force: + # delete_params = True + # project = args.base_project + # console.print( + # f"Deleting all child projects under parent {project['name']}, this includes " + + # "each project's parameters" + # ) + + # helpers.delete_project(project, _api_url, _headers, delete_params) + + # # reset org + # if args.reset_org and args.force: + # console.print('Nuking everything') + # # implement y/n + # foo = helpers.nuke() + # console.print('Nuke complete') + # exit(foo) + + # TODO: implement template create + # TODO: implement destroy_params + # TODO: implement destroy_envs + # TODO: implement destroy_projects + # TODO: implement destroy_all + # TODO: implement store data? + + +if __name__ == "__main__": + main() diff --git a/repro/many-nested.sh b/repro/many-nested.sh new file mode 100755 index 0000000..f0fc8c0 --- /dev/null +++ b/repro/many-nested.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +ROOT_NAME="project-count-limits" +MAX_DEPTH=5 +CHILDREN_PER_TIER=4 + +# Main script +main() { + local config_profile="$1" + local ct_type="$2" + local root="$ROOT_NAME" + local max_depth="$MAX_DEPTH" + local num_children="$CHILDREN_PER_TIER" + + echo "Creating root: $root" + cloudtruth --profile "$config_profile" "$ct_type" set "$root" + + declare -a current_depth_array + declare -a next_depth_array + current_depth_array=("$root") + + for (( depth = 1; depth <= max_depth; depth++ )); do + next_depth_array=() + + for current_item in "${current_depth_array[@]}"; do + for (( i = 1; i <= num_children; i++ )); do + local item="${current_item}_child${i}" + cloudtruth --profile "$config_profile" "$ct_type" set -p "$current_item" "$item" + next_depth_array+=("$item") + done + echo "Created/Updated $(( num_children )) items" + done + + current_depth_array=("${next_depth_array[@]}") + echo "Created/Updated $(( ${#current_depth_array[@]} )) items at depth $depth" + done + + echo "Created/Updated $(( num_children^max_depth )) items total" +} + +# Run the main script +## $1 = config_profile +## $2 = ct_type (projects or environments) +main "$@" diff --git a/repro/requirements.txt b/repro/requirements.txt new file mode 100644 index 0000000..cbdcaf0 --- /dev/null +++ b/repro/requirements.txt @@ -0,0 +1,4 @@ +PyYAML==6.0 +requests==2.28.2 +rich==13.3.3 +shortuuid==1.0.11