wip: make one script to rule aws/gcp

images/capi/hack/boskos.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Checks out a Image-Builder AWS account"""
+
+import argparse
+import json
+import os
+import time
+
+import requests
+
+BOSKOS_HOST = os.environ.get("BOSKOS_HOST", "boskos")
+BOSKOS_RESOURCE_NAME = os.environ.get('BOSKOS_RESOURCE_NAME')
+
+
+def checkout_account(resource_type, user, cloud, input_state="clean"):
+    url = f'http://{BOSKOS_HOST}/acquire'
+    data = {
+        'type': resource_type,
+        'owner': user,
+        'state': input_state,
+        'dest': 'busy',
+    }
+
+    r = requests.post(url, data)
+
+    if r.status_code == 404:
+      data['state'] = "free"
+      r = requests.post(url, data)
+
+    if r.status_code == 200:
+        content = r.content.decode()
+        result = json.loads(content)
+
+        print(f"export BOSKOS_RESOURCE_NAME={result['name']}")
+        if cloud.lower() == "aws":
+          print(f"export AWS_ACCESS_KEY_ID={result['userdata']['access-key-id']}")
+          print(f"export AWS_SECRET_ACCESS_KEY={result['userdata']['secret-access-key']}")
+        elif cloud.lower() == "gcp":
+          print(f"export GCP_PROJECT={result['name']}")
+
+
+    else:
+        raise Exception(f"Got invalid response {r.status_code}: {r.reason}")
+
+
+def release_account(user):
+    url = f'http://{BOSKOS_HOST}/release'
+    data = {
+        'name': BOSKOS_RESOURCE_NAME,
+        'owner': user,
+        'dest': 'dirty',
+    }
+
+    r = requests.post(url, data)
+    if r.status_code != 200:
+        raise Exception(f"Got invalid response {r.status_code}: {r.reason}")
+
+
+def send_heartbeat(user):
+    url = f'http://{BOSKOS_HOST}/update'
+    data = {
+        'name': BOSKOS_RESOURCE_NAME,
+        'owner': user,
+        'state': 'busy',
+    }
+
+    while True:
+        print(f"POST-ing heartbeat for resource {BOSKOS_RESOURCE_NAME} to {BOSKOS_HOST}")
+        r = requests.post(url, data)
+
+        if r.status_code == 200:
+            print(f"response status: {r.status_code}")
+        else:
+            print(f"Got invalid response {r.status_code}: {r.reason}")
+
+        time.sleep(60)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Boskos AWS Account Management')
+
+    parser.add_argument(
+        '--get', dest='checkout_account', action="store_true",
+        help='Checkout a Boskos AWS/GCP Account'
+    )
+
+    parser.add_argument(
+        '--release', dest='release_account', action="store_true",
+        help='Release a Boskos AWS/GCP Account'
+    )
+
+    parser.add_argument(
+        '--heartbeat', dest='send_heartbeat', action="store_true",
+        help='Send heartbeat for the checked out a Boskos AWS Account'
+    )
+
+    parser.add_argument(
+        '--resource-type', dest="resource_type", type=str,
+        required=True,
+        help="Type of Boskos resource to manage"
+    )
+
+    parser.add_argument(
+        '--user', dest="user", type=str,
+        default="sigs.k8s.io/image-builder",
+        help="username"
+    )
+
+    parser.add_argument(
+        '--cloud', dest="cloud", type=str,
+        required=True,
+        help="Which Cloud to use. AWS/GCP"
+    )
+
+    args = parser.parse_args()
+
+    if args.checkout_account:
+        checkout_account(args.resource_type, args.user, args.cloud)
+
+    elif args.release_account:
+        release_account(args.user)
+
+    elif args.send_heartbeat:
+        send_heartbeat(args.user)
+
+
+if __name__ == "__main__":
+    main()

images/capi/packer/gce/scripts/ci-gce.sh

@@ -48,7 +48,7 @@ if [ -n "${BOSKOS_HOST:-}" ]; then
   # Check out the account from Boskos and store the produced environment
   # variables in a temporary file.
   account_env_var_file="$(mktemp)"
-  python hack/checkout_account.py 1>"${account_env_var_file}"
+  python3 hack/boskos.py --get --resource-type "gce-project" --cloud "gcp" 1>"${account_env_var_file}"
   checkout_account_status="${?}"
 
   # If the checkout process was a success then load the account's
@@ -65,9 +65,7 @@ if [ -n "${BOSKOS_HOST:-}" ]; then
     exit "${checkout_account_status}"
   fi
 
-  # run the heart beat process to tell boskos that we are still
-  # using the checked out account periodically
-  python -u hack/heartbeat_account.py >> "$ARTIFACTS/logs/boskos.log" 2>&1 &
+  python3 -u hack/boskos.py --hearbeat >>$ARTIFACTS/boskos.log 2>&1 &
   HEART_BEAT_PID=$(echo $!)
 fi
 
@@ -94,13 +92,13 @@ EOF
   return 2
 fi
 
-export GCP_PROJECT_ID="$GCP_REGION"
+export GCP_PROJECT_ID="$GCP_PROJECT"
 
 make deps-gce
 make build-gce-all
 test_status="${?}"
 
 # If Boskos is being used then release the GCP project back to Boskos.
-[ -z "${BOSKOS_HOST:-}" ] || hack/checkin_account.py >> $ARTIFACTS/logs/boskos.log 2>&1
+[ -z "${BOSKOS_HOST:-}" ] || hack/boskos.py --release >> $ARTIFACTS/logs/boskos.log 2>&1
 
 exit "${test_status}"