diff --git a/images/capi/hack/boskos.py b/images/capi/hack/boskos.py new file mode 100644 index 0000000000..f0f1e52ec3 --- /dev/null +++ b/images/capi/hack/boskos.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks out a Image-Builder AWS account""" + +import argparse +import json +import os +import time + +import requests + +BOSKOS_HOST = os.environ.get("BOSKOS_HOST", "boskos") +BOSKOS_RESOURCE_NAME = os.environ.get('BOSKOS_RESOURCE_NAME') + + +def checkout_account(resource_type, user, cloud, input_state="clean"): + url = f'http://{BOSKOS_HOST}/acquire' + data = { + 'type': resource_type, + 'owner': user, + 'state': input_state, + 'dest': 'busy', + } + + r = requests.post(url, data) + + if r.status_code == 404: + data['state'] = "free" + r = requests.post(url, data) + + if r.status_code == 200: + content = r.content.decode() + result = json.loads(content) + + print(f"export BOSKOS_RESOURCE_NAME={result['name']}") + if cloud.lower() == "aws": + print(f"export AWS_ACCESS_KEY_ID={result['userdata']['access-key-id']}") + print(f"export AWS_SECRET_ACCESS_KEY={result['userdata']['secret-access-key']}") + elif cloud.lower() == "gcp": + print(f"export GCP_PROJECT={result['name']}") + + + else: + raise Exception(f"Got invalid response {r.status_code}: {r.reason}") + + +def release_account(user): + url = f'http://{BOSKOS_HOST}/release' + data = { + 'name': BOSKOS_RESOURCE_NAME, + 'owner': user, + 'dest': 'dirty', + } + + r = requests.post(url, data) + if r.status_code != 200: + raise Exception(f"Got invalid response {r.status_code}: {r.reason}") + + +def send_heartbeat(user): + url = f'http://{BOSKOS_HOST}/update' + data = { + 'name': BOSKOS_RESOURCE_NAME, + 'owner': user, + 'state': 'busy', + } + + while True: + print(f"POST-ing heartbeat for resource {BOSKOS_RESOURCE_NAME} to {BOSKOS_HOST}") + r = requests.post(url, data) + + if r.status_code == 200: + print(f"response status: {r.status_code}") + else: + print(f"Got invalid response {r.status_code}: {r.reason}") + + time.sleep(60) + + +def main(): + parser = argparse.ArgumentParser(description='Boskos AWS Account Management') + + parser.add_argument( + '--get', dest='checkout_account', action="store_true", + help='Checkout a Boskos AWS/GCP Account' + ) + + parser.add_argument( + '--release', dest='release_account', action="store_true", + help='Release a Boskos AWS/GCP Account' + ) + + parser.add_argument( + '--heartbeat', dest='send_heartbeat', action="store_true", + help='Send heartbeat for the checked out a Boskos AWS Account' + ) + + parser.add_argument( + '--resource-type', dest="resource_type", type=str, + required=True, + help="Type of Boskos resource to manage" + ) + + parser.add_argument( + '--user', dest="user", type=str, + default="sigs.k8s.io/image-builder", + help="username" + ) + + parser.add_argument( + '--cloud', dest="cloud", type=str, + required=True, + help="Which Cloud to use. AWS/GCP" + ) + + args = parser.parse_args() + + if args.checkout_account: + checkout_account(args.resource_type, args.user, args.cloud) + + elif args.release_account: + release_account(args.user) + + elif args.send_heartbeat: + send_heartbeat(args.user) + + +if __name__ == "__main__": + main() diff --git a/images/capi/hack/checkin_account.py b/images/capi/hack/checkin_account.py deleted file mode 100755 index dc4dd12616..0000000000 --- a/images/capi/hack/checkin_account.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import httplib -import urllib -import os - -BOSKOS_HOST=os.environ.get("BOSKOS_HOST", "boskos") -BOSKOS_RESOURCE_NAME=os.environ['BOSKOS_RESOURCE_NAME'] - -USER = "image-builder" - -if __name__ == "__main__": - conn = httplib.HTTPConnection(BOSKOS_HOST) - conn.request("POST", "/release?%s" % urllib.urlencode({ - 'name': BOSKOS_RESOURCE_NAME, - 'dest': 'dirty', - 'owner': USER, - })) - - resp = conn.getresponse() - if resp.status != 200: - sys.exit("Got invalid response %d" % resp.status) - conn.close() diff --git a/images/capi/hack/checkout_account.py b/images/capi/hack/checkout_account.py deleted file mode 100755 index b4810f3ed8..0000000000 --- a/images/capi/hack/checkout_account.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Checks out a gcp account from E2E""" - -import urllib -import httplib -import os -import sys -import json - -BOSKOS_HOST=os.environ.get("BOSKOS_HOST", "boskos") - -RESOURCE_TYPE = "gce-project" -USER = "image-builder" - -def post_request(host, input_state = "clean"): - conn = httplib.HTTPConnection(host) - conn.request("POST", "/acquire?%s" % urllib.urlencode({ - 'type': RESOURCE_TYPE, - 'owner': USER, - 'state': input_state, - 'dest': 'busy', - }) - - ) - resp = conn.getresponse() - status = resp.status - reason = resp.reason - body = resp.read() - conn.close() - return status, reason, body - -if __name__ == "__main__": - status, reason, result = post_request(BOSKOS_HOST) - # we're working around an issue with the data in boskos. - # We'll remove the code that tries both free and clean once all the data is good. - # Afterwards we should just check for free - if status == 404: - status, reason, result = post_request(BOSKOS_HOST, "free") - - if status != 200: - sys.exit("Got invalid response %d: %s" % (status, reason)) - - body = json.loads(result) - print 'export BOSKOS_RESOURCE_NAME="%s";' % body['name'] - print 'export GCP_PROJECT="%s";' % body['name'] diff --git a/images/capi/hack/heartbeat_account.py b/images/capi/hack/heartbeat_account.py deleted file mode 100755 index 68c82eb774..0000000000 --- a/images/capi/hack/heartbeat_account.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2019 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import httplib -import urllib -import os -import time - -BOSKOS_HOST=os.environ.get("BOSKOS_HOST", "boskos") -BOSKOS_RESOURCE_NAME=os.environ['BOSKOS_RESOURCE_NAME'] - -USER = "image-builder" - -if __name__ == "__main__": - count = 0 - # keep sending heart beat for 3 hours - while count < 180: - conn = httplib.HTTPConnection(BOSKOS_HOST) - conn.request("POST", "/update?%s" % urllib.urlencode({ - 'name': BOSKOS_RESOURCE_NAME, - 'state': 'busy', - 'owner': USER, - })) - - resp = conn.getresponse() - if resp.status != 200: - sys.exit("Got invalid response %d : %r : %r" % (resp.status, resp.reason, resp)) - conn.close() - # sleep for a minute - time.sleep(60) diff --git a/images/capi/packer/gce/scripts/ci-gce.sh b/images/capi/packer/gce/scripts/ci-gce.sh index ec6a846da4..9fc8de52cd 100644 --- a/images/capi/packer/gce/scripts/ci-gce.sh +++ b/images/capi/packer/gce/scripts/ci-gce.sh @@ -48,7 +48,7 @@ if [ -n "${BOSKOS_HOST:-}" ]; then # Check out the account from Boskos and store the produced environment # variables in a temporary file. account_env_var_file="$(mktemp)" - python hack/checkout_account.py 1>"${account_env_var_file}" + python3 hack/boskos.py --get --resource-type "gce-project" --cloud "gcp" 1>"${account_env_var_file}" checkout_account_status="${?}" # If the checkout process was a success then load the account's @@ -65,9 +65,7 @@ if [ -n "${BOSKOS_HOST:-}" ]; then exit "${checkout_account_status}" fi - # run the heart beat process to tell boskos that we are still - # using the checked out account periodically - python -u hack/heartbeat_account.py >> "$ARTIFACTS/logs/boskos.log" 2>&1 & + python3 -u hack/boskos.py --hearbeat >>$ARTIFACTS/boskos.log 2>&1 & HEART_BEAT_PID=$(echo $!) fi @@ -94,13 +92,13 @@ EOF return 2 fi -export GCP_PROJECT_ID="$GCP_REGION" +export GCP_PROJECT_ID="$GCP_PROJECT" make deps-gce make build-gce-all test_status="${?}" # If Boskos is being used then release the GCP project back to Boskos. -[ -z "${BOSKOS_HOST:-}" ] || hack/checkin_account.py >> $ARTIFACTS/logs/boskos.log 2>&1 +[ -z "${BOSKOS_HOST:-}" ] || hack/boskos.py --release >> $ARTIFACTS/logs/boskos.log 2>&1 exit "${test_status}"