diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md
index 92a0f068..33a68541 100644
--- a/.github/CODE_OF_CONDUCT.md
+++ b/.github/CODE_OF_CONDUCT.md
@@ -68,7 +68,7 @@ offensive, or harmful.
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
+representing a project or community include using an official project email
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
@@ -76,9 +76,9 @@ further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the EGI Foundation team at contact@egi.eu. The team will
-review and investigate all complaints, and will respond in a way that it deems
-appropriate to the circumstances. The team is obligated to maintain
+reported by contacting the [EGI Foundation team](mailto:contact@egi.eu). The
+team will review and investigate all complaints, and will respond in a way that
+it deems appropriate to the circumstances. The team is obligated to maintain
confidentiality with regard to the reporter of an incident. Further details of
specific enforcement policies may be posted separately.
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
index b5640659..d0ae11ab 100644
--- a/.github/workflows/lint.yml
+++ b/.github/workflows/lint.yml
@@ -1,7 +1,7 @@
---
name: Lint
-on: [pull_request, push]
+on: pull_request
jobs:
super-lint:
@@ -12,9 +12,15 @@ jobs:
# Checks out a copy of your repository on the ubuntu-latest machine
- name: Checkout code
uses: actions/checkout@v4
+ with:
+ # Full git history needed to get proper list of changed files
+ fetch-depth: 0
# Runs the Super-Linter action
- name: Run Super-Linter
- uses: github/super-linter/slim@v5
+ uses: github/super-linter/slim@v6
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ # Disabling these two for the moment, should be enabled later
+ VALIDATE_CHECKOV: false
+ VALIDATE_PYTHON_PYLINT: false
diff --git a/cloud-info/ams-wrapper.sh b/cloud-info/ams-wrapper.sh
index b28a758d..7f1c0b77 100755
--- a/cloud-info/ams-wrapper.sh
+++ b/cloud-info/ams-wrapper.sh
@@ -10,9 +10,9 @@ GOCDB_ID=$(python -c "from __future__ import print_function; \
timeout=60)['gocdb_id'], end='')")
if test "$AMS_TOKEN_FILE" != ""; then
- AMS_TOKEN=$(cat "$AMS_TOKEN_FILE")
-elif test "$HOSTCERT" != "" -a "$HOSTKEY" != ""; then
- AMS_TOKEN=$(python -c "from argo_ams_library import ArgoMessagingService; \
+ AMS_TOKEN=$(cat "$AMS_TOKEN_FILE")
+elif test "$HOSTCERT" != "" -a "$HOSTKEY" != ""; then
+ AMS_TOKEN=$(python -c "from argo_ams_library import ArgoMessagingService; \
ams = ArgoMessagingService(endpoint='$AMS_HOST', \
project='$AMS_PROJECT', \
cert='$HOSTCERT', \
@@ -21,16 +21,18 @@ elif test "$HOSTCERT" != "" -a "$HOSTKEY" != ""; then
fi
if test "$SITE_NAME" = ""; then
- SITE_NAME="$(yq -r .site.name "$CLOUD_INFO_CONFIG" | tr "." "-")"
+ SITE_NAME="$(yq -r .site.name "$CLOUD_INFO_CONFIG" | tr "." "-")"
fi
SITE_TOPIC=$(echo "$SITE_NAME" | tr "." "-")
AMS_TOPIC="SITE_${SITE_TOPIC}_ENDPOINT_${GOCDB_ID}"
# exit if TOPIC is not available.
-curl -f "https://$AMS_HOST/v1/projects/$AMS_PROJECT/topics/$AMS_TOPIC?key=$AMS_TOKEN" > /dev/null 2>&1 \
- || (echo "Topic $AMS_TOPIC is not avaiable, aborting!"; false)
-
+curl -f "https://$AMS_HOST/v1/projects/$AMS_PROJECT/topics/$AMS_TOPIC?key=$AMS_TOKEN" >/dev/null 2>&1 ||
+ (
+ echo "Topic $AMS_TOPIC is not avaiable, aborting!"
+ false
+ )
# Attempt to generate the site configuration
AUTO_CONFIG_PATH="$(mktemp -d)"
@@ -41,57 +43,60 @@ export CHECKIN_SECRETS_FILE="$CHECKIN_SECRETS_PATH/secrets.yaml"
export ACCESS_TOKEN_FILE="$AUTO_CONFIG_PATH/token.yaml"
USE_ACCESS_TOKEN=0
if token-generator; then
- # TODO(enolfc): even if this belows fails, we should use access token as it will provide
- # access to more projects
- if SECRETS_FILE="$ACCESS_TOKEN_FILE" config-generator > "$AUTO_CONFIG_PATH/site.yaml"; then
- # this worked, let's update the env
- export CHECKIN_SECRETS_PATH="$AUTO_CONFIG_PATH/vos"
- export CLOUD_INFO_CONFIG="$AUTO_CONFIG_PATH/site.yaml"
- USE_ACCESS_TOKEN=1
- fi
+ # TODO(enolfc): even if this belows fails, we should use access token as it will provide
+ # access to more projects
+ if SECRETS_FILE="$ACCESS_TOKEN_FILE" config-generator >"$AUTO_CONFIG_PATH/site.yaml"; then
+ # this worked, let's update the env
+ export CHECKIN_SECRETS_PATH="$AUTO_CONFIG_PATH/vos"
+ export CLOUD_INFO_CONFIG="$AUTO_CONFIG_PATH/site.yaml"
+ USE_ACCESS_TOKEN=1
+ fi
fi
# Any OS related parameter should be available as env variables
if test "$CHECKIN_SECRETS_PATH" = ""; then
- # Case 1: manual config
- cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \
- --middleware "$CLOUD_INFO_MIDDLEWARE" \
- --ignore-share-errors \
- --format glue21 > cloud-info.out
+ # Case 1: manual config
+ cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \
+ --middleware "$CLOUD_INFO_MIDDLEWARE" \
+ --ignore-share-errors \
+ --format glue21 >cloud-info.out
elif test "$USE_ACCESS_TOKEN" -eq 1; then
- # Case 2: access token style
- cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \
- --middleware "$CLOUD_INFO_MIDDLEWARE" \
- --ignore-share-errors \
- --auth-refresher accesstoken \
- --format glue21 > cloud-info.out
+ # Case 2: access token style
+ cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \
+ --middleware "$CLOUD_INFO_MIDDLEWARE" \
+ --ignore-share-errors \
+ --auth-refresher accesstoken \
+ --format glue21 >cloud-info.out
else
- # Let's use the service account directly on the info provider
- CHECKIN_DISCOVERY="https://aai.egi.eu/auth/realms/egi/.well-known/openid-configuration"
- CLIENT_ID="$(yq -r '.fedcloudops.client_id' < "$CHECKIN_SECRETS_FILE")"
- CLIENT_SECRET="$(yq -r '.fedcloudops.client_secret' < "$CHECKIN_SECRETS_FILE")"
- cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \
- --middleware "$CLOUD_INFO_MIDDLEWARE" \
- --ignore-share-errors \
- --os-auth-type v3oidcclientcredentials \
- --os-discovery-endpoint "$CHECKIN_DISCOVERY" \
- --os-client-id "$CLIENT_ID" \
- --os-client-secret "$CLIENT_SECRET" \
- --os-access-token-type access_token \
- --os-openid-scope "openid profile eduperson_entitlement email" \
- --format glue21 > cloud-info.out
+ # Let's use the service account directly on the info provider
+ CHECKIN_DISCOVERY="https://aai.egi.eu/auth/realms/egi/.well-known/openid-configuration"
+ CLIENT_ID="$(yq -r '.fedcloudops.client_id' <"$CHECKIN_SECRETS_FILE")"
+ CLIENT_SECRET="$(yq -r '.fedcloudops.client_secret' <"$CHECKIN_SECRETS_FILE")"
+ cloud-info-provider-service --yaml-file "$CLOUD_INFO_CONFIG" \
+ --middleware "$CLOUD_INFO_MIDDLEWARE" \
+ --ignore-share-errors \
+ --os-auth-type v3oidcclientcredentials \
+ --os-discovery-endpoint "$CHECKIN_DISCOVERY" \
+ --os-client-id "$CLIENT_ID" \
+ --os-client-secret "$CLIENT_SECRET" \
+ --os-access-token-type access_token \
+ --os-openid-scope "openid profile eduperson_entitlement email" \
+ --format glue21 >cloud-info.out
fi
# Fail if there are no shares
-grep -q GLUE2ShareID cloud-info.out \
- || (echo "No share information available, aborting!"; false)
+grep -q GLUE2ShareID cloud-info.out ||
+ (
+ echo "No share information available, aborting!"
+ false
+ )
# Publishing on our own as message is too large for some providers
ARGO_URL="https://$AMS_HOST/v1/projects/$AMS_PROJECT/topics/$AMS_TOPIC:publish?key=$AMS_TOKEN"
-printf '{"messages":[{"attributes":{},"data":"' > ams-payload
-grep -v "UNKNOWN" cloud-info.out | grep -v "^#" | grep -v ": $" | gzip | base64 -w 0 >> ams-payload
-printf '"}]}' >> ams-payload
+printf '{"messages":[{"attributes":{},"data":"' >ams-payload
+grep -v "UNKNOWN" cloud-info.out | grep -v "^#" | grep -v ": $" | gzip | base64 -w 0 >>ams-payload
+printf '"}]}' >>ams-payload
curl -X POST "$ARGO_URL" -H "content-type: application/json" -d @ams-payload
diff --git a/cloud-info/publisher.sh b/cloud-info/publisher.sh
index fe89b88f..0ba9e979 100755
--- a/cloud-info/publisher.sh
+++ b/cloud-info/publisher.sh
@@ -10,9 +10,9 @@ export CLOUD_INFO_MIDDLEWARE=openstack
ams-wrapper.sh
if [ -n "$OCCI_ENDPOINT" ]; then
- # OCCI
- export GOCDB_URL="$OCCI_ENDPOINT"
- export GOCDB_SERVICE_TYPE=eu.egi.cloud.vm-management.occi
- export CLOUD_INFO_MIDDLEWARE=ooi
- ams-wrapper.sh
+ # OCCI
+ export GOCDB_URL="$OCCI_ENDPOINT"
+ export GOCDB_SERVICE_TYPE=eu.egi.cloud.vm-management.occi
+ export CLOUD_INFO_MIDDLEWARE=ooi
+ ams-wrapper.sh
fi
diff --git a/deploy/deploy.sh b/deploy/deploy.sh
index eaff5587..2e7ab19b 100755
--- a/deploy/deploy.sh
+++ b/deploy/deploy.sh
@@ -11,21 +11,21 @@ SLACK_WEBHOOK_URL="$4"
ansible-galaxy install git+https://github.com/EGI-Federation/ansible-role-fedcloud-ops.git
-echo "cloud_info_image: \"ghcr.io/egi-federation/fedcloud-cloud-info:sha-$SHORT_SHA\"" >> extra-vars.yaml
+echo "cloud_info_image: \"ghcr.io/egi-federation/fedcloud-cloud-info:sha-$SHORT_SHA\"" >>extra-vars.yaml
# Configure!
-if ansible-playbook -i inventory.yaml \
- --extra-vars @secrets.yaml \
- --extra-vars @extra-vars.yaml \
- --extra-vars @vos.yaml \
- playbook.yaml >ansible.log 2>&1 ; then
- status_summary="success"
- color="#6DBF59"
- header="Successful deployment :rocket:"
+if ansible-playbook -i inventory.yaml \
+ --extra-vars @secrets.yaml \
+ --extra-vars @extra-vars.yaml \
+ --extra-vars @vos.yaml \
+ playbook.yaml >ansible.log 2>&1; then
+ status_summary="success"
+ color="#6DBF59"
+ header="Successful deployment :rocket:"
else
- status_summary="fail"
- color="#EA4F47"
- header="Failed deployment :boom:"
+ status_summary="fail"
+ color="#EA4F47"
+ header="Failed deployment :boom:"
fi
# This is a temporary way to get the auto discovery working while we transition for all sites
@@ -36,32 +36,32 @@ GITHUB_COMMIT_URL="https://api.github.com/repos/EGI-Federation/fedcloud-catchall
# Find out PR we need to update
ISSUE_NUMBER=$(curl \
- -H "Accept: application/vnd.github.groot-preview+json" \
- "$GITHUB_COMMIT_URL" | jq .[0].number)
+ -H "Accept: application/vnd.github.groot-preview+json" \
+ "$GITHUB_COMMIT_URL" | jq .[0].number)
GITHUB_ISSUE_URL="https://api.github.com/repos/EGI-Federation/fedcloud-catchall-operations/issues/$ISSUE_NUMBER/comments"
{
- echo "### Ansible deployment: \`$status_summary\`"
- echo 'Deployment log
'
- echo
- echo '```'
- cat ansible.log
- echo '```'
- echo
- echo ' '
-} > github_body.txt
-echo "{}" | jq --arg b "$(cat github_body.txt)" '{body: $b}' > github_body.json
+ echo "### Ansible deployment: \`$status_summary\`"
+ echo 'Deployment log
'
+ echo
+ echo '```'
+ cat ansible.log
+ echo '```'
+ echo
+ echo ' '
+} >github_body.txt
+echo "{}" | jq --arg b "$(cat github_body.txt)" '{body: $b}' >github_body.json
# Let GitHub know
comment_url=$(curl -X POST \
- -H "Authorization: token $OAUTH_TOKEN" \
- -H "Accept: application/vnd.github.v3+json" \
- "$GITHUB_ISSUE_URL" \
- --data @github_body.json | \
- jq -r .html_url)
+ -H "Authorization: token $OAUTH_TOKEN" \
+ -H "Accept: application/vnd.github.v3+json" \
+ "$GITHUB_ISSUE_URL" \
+ --data @github_body.json |
+ jq -r .html_url)
-cat > slack_body.json << EOF
+cat >slack_body.json < "$VO_LIST"
+curl --silent "http://cclavoisier01.in2p3.fr:8080/lavoisier/VoList?accept=json" |
+ jq -r ".data[].name" >"$VO_LIST"
# Get fedcloudclient sites
FEDCLOUD_CLI_SITES=$(mktemp)
curl "https://raw.githubusercontent.com/tdviet/fedcloudclient/master/config/sites.yaml" \
- > "$FEDCLOUD_CLI_SITES"
+ >"$FEDCLOUD_CLI_SITES"
# Temp file for nova endpoint
NOVA_ENDPOINT=$(mktemp)
-for f in sites/*.yaml
-do
- goc_site=$(grep "^gocdb:" "$f" | cut -f2 -d":" | tr -d "[:space:]")
- endpoint=$(grep "^endpoint:" "$f" | cut -f2- -d":" | tr -d "[:space:]")
- printf "Searching for endpoint %s in %s site (%s)\n" \
- "$endpoint" "$goc_site" "$f"
- curl --silent "$goc_method&sitename=$goc_site&service_type=org.openstack.nova" \
- > "$NOVA_ENDPOINT"
- if ! grep -q "$goc_site" "$NOVA_ENDPOINT"
- then
- printf "\033[0;31m[ERROR] Site %s not found in GOC\033[0m\n" "$goc_site"
- exit_value=1
- continue
- fi
- if ! grep -q "$endpoint" "$NOVA_ENDPOINT"
- then
- printf "\033[0;31m[ERROR] URL %s for %s not found in GOC\033[0m\n" \
- "$endpoint" "$goc_site"
- exit_value=1
- else
- printf "\033[0;32m[OK]\033[0m\n"
- fi
- # check if all VOs configured do exist
- # Try to use FQAN
- # So the VO that comes from the file, it will be either:
- # - just the name of the VO
- # - //some more extra/
- # - /VO=/some more stuff/
- for vo in $(yq -r ".vos[].name" < "$f" | cut -f2 -d"/" | sed "s/^VO=//")
- do
- if ! grep -q "^$vo\$" "$VO_LIST"
- then
- printf "\033[0;31m[ERROR] VO %s not found in ops portal\033[0m\n" \
- "$vo"
- exit_value=1
- fi
- done
+for f in sites/*.yaml; do
+ goc_site=$(grep "^gocdb:" "$f" | cut -f2 -d":" | tr -d "[:space:]")
+ endpoint=$(grep "^endpoint:" "$f" | cut -f2- -d":" | tr -d "[:space:]")
+ printf "Searching for endpoint %s in %s site (%s)\n" \
+ "$endpoint" "$goc_site" "$f"
+ curl --silent "$goc_method&sitename=$goc_site&service_type=org.openstack.nova" \
+ >"$NOVA_ENDPOINT"
+ if ! grep -q "$goc_site" "$NOVA_ENDPOINT"; then
+ printf "\033[0;31m[ERROR] Site %s not found in GOC\033[0m\n" "$goc_site"
+ exit_value=1
+ continue
+ fi
+ if ! grep -q "$endpoint" "$NOVA_ENDPOINT"; then
+ printf "\033[0;31m[ERROR] URL %s for %s not found in GOC\033[0m\n" \
+ "$endpoint" "$goc_site"
+ exit_value=1
+ else
+ printf "\033[0;32m[OK]\033[0m\n"
+ fi
+ # check if all VOs configured do exist
+ # Try to use FQAN
+ # So the VO that comes from the file, it will be either:
+ # - just the name of the VO
+ # - //some more extra/
+ # - /VO=/some more stuff/
+ for vo in $(yq -r ".vos[].name" <"$f" | cut -f2 -d"/" | sed "s/^VO=//"); do
+ if ! grep -q "^$vo\$" "$VO_LIST"; then
+ printf "\033[0;31m[ERROR] VO %s not found in ops portal\033[0m\n" \
+ "$vo"
+ exit_value=1
+ fi
+ done
- # check if site is also on:
- # https://github.com/tdviet/fedcloudclient/blob/master/config/sites.yaml
- if ! grep -q "$f" "$FEDCLOUD_CLI_SITES"
- then
- printf "\033[0;31m[ERROR] Site %s not found in fedcloudclient\033[0m\n" "$goc_site"
- exit_value=1
- fi
+ # check if site is also on:
+ # https://github.com/tdviet/fedcloudclient/blob/master/config/sites.yaml
+ if ! grep -q "$f" "$FEDCLOUD_CLI_SITES"; then
+ printf "\033[0;31m[ERROR] Site %s not found in fedcloudclient\033[0m\n" "$goc_site"
+ exit_value=1
+ fi
done
-for site in $(yq -r '.[]' < "$FEDCLOUD_CLI_SITES")
-do
- if ! test -s "sites/$(basename "$site")"
- then
- printf "\033[0;31m[ERROR] Site %s not found in fedcloud-catchall-operations\033[0m\n" "$(basename "$site")"
- exit_value=1
- fi
+for site in $(yq -r '.[]' <"$FEDCLOUD_CLI_SITES"); do
+ if ! test -s "sites/$(basename "$site")"; then
+ printf "\033[0;31m[ERROR] Site %s not found in fedcloud-catchall-operations\033[0m\n" "$(basename "$site")"
+ exit_value=1
+ fi
done
# check that the VO mappings are up to date according to ops portal
-for vo in $(yq -r '.vos | keys[]' < vo-mappings.yaml | cut -f2 -d"/" | sed "s/^VO=//")
-do
- if ! grep -q "^$vo\$" "$VO_LIST"
- then
- printf "\033[0;31m[ERROR] VO %s not found in ops portal\033[0m\n" \
- "$vo"
- exit_value=1
- fi
+for vo in $(yq -r '.vos | keys[]'