From e48c93f0a000d63735f05e98a991479520dbacc2 Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Fri, 3 Dec 2021 16:42:50 +0100 Subject: [PATCH 1/4] Clean repository of files that will migrate to template repo Signed-off-by: Brend Smits --- .github/workflows/environment.yml | 2 +- .github/workflows/lint-batch.yml | 66 --------------- .github/workflows/lint-using-action.yml | 4 +- .github/workflows/lint.yml | 102 ------------------------ .github/workflows/run-loop-on-cron.yml | 21 ----- bin/check-tracked-repo.sh | 12 --- bin/compare-timeout-times.sh | 19 ----- data/checked-repos.txt | 0 data/gh-repos.txt | 0 data/repo.txt | 1 - 10 files changed, 3 insertions(+), 224 deletions(-) delete mode 100644 .github/workflows/lint-batch.yml delete mode 100644 .github/workflows/lint.yml delete mode 100644 .github/workflows/run-loop-on-cron.yml delete mode 100644 bin/check-tracked-repo.sh delete mode 100644 bin/compare-timeout-times.sh delete mode 100644 data/checked-repos.txt delete mode 100644 data/gh-repos.txt delete mode 100644 data/repo.txt diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index acee292..eb6f629 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -23,7 +23,7 @@ jobs: dockerfile: . image-name: continuous-compliance tags: latest 0.1 - push-branches: main develop feature/multiple-repos-input + push-branches: main develop env: DOCKER_USERNAME: ${{ github.actor }} DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/lint-batch.yml b/.github/workflows/lint-batch.yml deleted file mode 100644 index 8b91630..0000000 --- a/.github/workflows/lint-batch.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: Repolinter-batch - -on: - workflow_dispatch: - inputs: - repositories_input: - description: 'Target Repositories (my-org/repository,my-org/repository-2)' - required: true - -jobs: - lint: - runs-on: ubuntu-latest - container: ghcr.io/philips-labs/continuous-compliance:latest - env: - TARGET_REPOS: ${{ github.event.inputs.repositories_input }} - defaults: - run: - shell: bash - - steps: - - name: Checkout CC for the loop script - uses: actions/checkout@v2 - - - name: Set variables - run: | - current_time=$(date +%s) - echo "::set-output name=CURRENT_TIME::$current_time" - id: set_variables - - - name: Run repolinter on target repository - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - for i in $(echo $TARGET_REPOS | sed "s/,/ /g") - do - export TARGET_REPO=$i - echo " checking: $i" - /app/repolinter/bin/repolinter.js lint \ - --rulesetFile ${GITHUB_WORKSPACE}/config/repolint.json \ - -g https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${i} - done - - - name: Git pull latest version of the checked-repos file to avoid conflicts - run: | - git pull - - - name: Update checked repo list - run: | - for i in $(echo $TARGET_REPOS | sed "s/,/ /g") - do - echo " update database: $i" - repo=${i} - escaped=${repo/\//\\/} - sed "/^$escaped/d" -i data/checked-repos.txt - echo ${i} ${{ steps.set_variables.outputs.CURRENT_TIME }} >> data/checked-repos.txt - done - - - name: Git commit & push the updated checked-repos.txt - continue-on-error: true - run: | - git config --global user.name 'Continuous Compliance' - git config --global user.email 'cc@philips-labs.com' - git add data/checked-repos.txt - git commit -m "Automatically added: ${{ steps.set_variables.outputs.REPO }} ${{ steps.set_variables.outputs.CURRENT_TIME }}" - git pull - git push diff --git a/.github/workflows/lint-using-action.yml b/.github/workflows/lint-using-action.yml index 662dad0..7168e0f 100644 --- a/.github/workflows/lint-using-action.yml +++ b/.github/workflows/lint-using-action.yml @@ -5,7 +5,7 @@ on: inputs: target_repos: description: 'Target Repositories (my-org/repository,my-org/repository-2)' - required: false + required: true jobs: run-linter: @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Run Linter - uses: philips-labs/continuous-compliance@feature/multiple-repos-input + uses: philips-labs/continuous-compliance@main env: target_repos: ${{ github.event.inputs.target_repos }} GITHUB_TOKEN: ${{ secrets.TEMP_TOKEN }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index c396fcc..0000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,102 +0,0 @@ -name: Repolinter - -on: - workflow_dispatch: - inputs: - repository_input: - description: 'Target Repository (my-org/repository)' - required: true - check_tracked: - description: 'Check if repository is checked before (true/false)' - required: true - default: 'true' - timeframe: - description: 'Timeframe in seconds' - required: false - default: '86400' - modify_tracked_list: - description: 'Upsert repository in tracked repo list (true/false)' - required: true - default: 'true' - -jobs: - lint: - runs-on: ubuntu-latest - container: ghcr.io/philips-labs/continuous-compliance:latest - env: - TARGET_REPO: ${{ github.event.inputs.repository_input }} - defaults: - run: - shell: bash - - steps: - - name: Checkout CC for the loop script - uses: actions/checkout@v2 - - - name: Run Bash script to check if repo is in Tracked file - if: ${{ github.event.inputs.check_tracked == 'true' }} - run: | - chmod +x bin/check-tracked-repo.sh - repo=$(bin/check-tracked-repo.sh data/checked-repos.txt ${{env.TARGET_REPO}}) - echo "::set-output name=TRACKED_REPO::$repo" - echo "::set-output name=REPO::$(echo $repo | cut -d' ' -f1)" - echo "::set-output name=TIME::$(echo $repo | cut -d' ' -f2)" - id: repo_tracker - - - name: Run Bash script to check if repo is within timeframe - if: ${{ github.event.inputs.check_tracked == 'true' && steps.repo_tracker.outputs.TRACKED_REPO != ''}} - run: | - chmod +x bin/compare-timeout-times.sh - check_repo=$(bin/compare-timeout-times.sh ${{ github.event.inputs.timeframe }} ${{ steps.repo_tracker.outputs.TIME }}) - echo "::set-output name=BOOLEAN::$check_repo" - echo $check_repo - id: check_repo - - - name: Set variables - run: | - time=${{ steps.repo_tracker.outputs.TIME }} - repo=${{ steps.repo_tracker.outputs.REPO }} - current_time=$(date +%s) - boolean=${{ steps.check_repo.outputs.BOOLEAN }} - target_repo=${{env.TARGET_REPO}} - echo "::set-output name=TIME::${time:=$(date +%s)}" - echo "::set-output name=CURRENT_TIME::$current_time" - echo "::set-output name=REPO::${repo:=$target_repo}" - echo "::set-output name=CHECK_REPO::${boolean:=true}" - echo "::set-output name=BRANCH_NAME::$(echo ${GITHUB_REF#refs/heads/})" - id: set_variables - - - name: Run repolinter on target repository - if: ${{ github.event.inputs.check_tracked == 'false' || steps.set_variables.outputs.CHECK_REPO == 'true' }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - /app/repolinter/bin/repolinter.js lint --rulesetFile ${GITHUB_WORKSPACE}/config/repolint.json -g https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{env.TARGET_REPO}} - - - name: Git pull latest version of the checked-repos file to avoid conflicts - if: ${{ github.event.inputs.modify_tracked_list == 'true' && steps.set_variables.outputs.CHECK_REPO == 'true' }} - run: | - git pull - - - name: Remove tracked repo from checked repo list - if: ${{ github.event.inputs.modify_tracked_list == 'true' && steps.set_variables.outputs.CHECK_REPO == 'true' }} - run: | - repo=${{ steps.set_variables.outputs.REPO }} - escaped=${repo/\//\\/} - sed "/$escaped ${{ steps.set_variables.outputs.TIME }}/d" -i data/checked-repos.txt - - - name: Add repo and timestamp to checked repo list - if: ${{ github.event.inputs.modify_tracked_list == 'true' && steps.set_variables.outputs.CHECK_REPO == 'true' }} - run: | - echo ${{ steps.set_variables.outputs.REPO }} ${{ steps.set_variables.outputs.CURRENT_TIME }} >> data/checked-repos.txt - - - name: Git commit & push the updated checked-repos.txt - if: ${{ github.event.inputs.modify_tracked_list == 'true' && steps.set_variables.outputs.CHECK_REPO == 'true' }} - continue-on-error: true - run: | - git config --global user.name 'Continuous Compliance' - git config --global user.email 'cc@philips-labs.com' - git add data/checked-repos.txt - git commit -m "Automatically added: ${{ steps.set_variables.outputs.REPO }} ${{ steps.set_variables.outputs.CURRENT_TIME }}" - git pull - git push diff --git a/.github/workflows/run-loop-on-cron.yml b/.github/workflows/run-loop-on-cron.yml deleted file mode 100644 index 11a99e3..0000000 --- a/.github/workflows/run-loop-on-cron.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Trigger repolinter on Cron job - -on: - workflow_dispatch: - -jobs: - trigger-loop: - runs-on: ubuntu-latest - defaults: - run: - shell: bash - - steps: - - name: Checkout CC for the loop script - uses: actions/checkout@v2 - - - name: Run Loop Script - run: | - bin/loop.sh data/gh-repos.txt 20 604800 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/bin/check-tracked-repo.sh b/bin/check-tracked-repo.sh deleted file mode 100644 index 03d6b9a..0000000 --- a/bin/check-tracked-repo.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -[ -n "$1" ] && TRACKED_REPO_LIST=$1 || [ -r "$2" ] && REPO=$2 || exit 1 - -while IFS="" read -r p || [ -n "$p" ] -do - REPO_FOUND=( $p ) - if [ "${REPO_FOUND}" = "${REPO}" ]; then - echo $p - break - fi -done < "$TRACKED_REPO_LIST" diff --git a/bin/compare-timeout-times.sh b/bin/compare-timeout-times.sh deleted file mode 100644 index 6faefd9..0000000 --- a/bin/compare-timeout-times.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -[ -n "$1" ] && TIMEOUT_SECONDS=$1 || [ -r "$2" ] && TARGET_REPO_EPOCH_TIME=$2 || exit 1 - -current_time=$(date +%s) -timeout_time=$(expr "$current_time" - "$TIMEOUT_SECONDS") - -# if the target repository time is larger or the same than the time out time, we should not do anything as -# the repo is within the timeframe and does not need checking. -# In this case we should exit - if (($TARGET_REPO_EPOCH_TIME >= $timeout_time)); then - echo 'false' - exit - else -# the repo is not within the timeframe and does need checking. -# In this case we should echo something and exit the application. - echo 'true' - exit - fi \ No newline at end of file diff --git a/data/checked-repos.txt b/data/checked-repos.txt deleted file mode 100644 index e69de29..0000000 diff --git a/data/gh-repos.txt b/data/gh-repos.txt deleted file mode 100644 index e69de29..0000000 diff --git a/data/repo.txt b/data/repo.txt deleted file mode 100644 index 484d6eb..0000000 --- a/data/repo.txt +++ /dev/null @@ -1 +0,0 @@ -philips-software/spdx-action From c5aba66ef3433a15e0f425926db92e981c388e8b Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Mon, 7 Feb 2022 12:16:00 +0100 Subject: [PATCH 2/4] Rename ruleset_file_url to ruleset Signed-off-by: Brend Smits --- action.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/action.yaml b/action.yaml index d26ae0c..0d0ac26 100644 --- a/action.yaml +++ b/action.yaml @@ -4,10 +4,9 @@ branding: icon: lock color: purple inputs: - ruleset_file_url: - description: 'url that links to a ruleset config file' + ruleset: + description: 'file path or url that links to a ruleset config file' required: true - default: 'https://gist.githubusercontent.com/Brend-Smits/385824d3422c0c06323bb1412cd5ba96/raw/08676dbf366a86641dd30fd5511b91afdee65adc/gistfile1.txt' gh_token: description: 'Github token that has permissions to create labels, issues and has read rights to view files.' required: true @@ -19,5 +18,5 @@ runs: image: 'docker://ghcr.io/philips-labs/continuous-compliance:latest' args: - "${{ inputs.gh_token }}" - - "${{ inputs.ruleset_file_url }}" + - "${{ inputs.ruleset }}" - "${{ inputs.target_repos }}" \ No newline at end of file From 02ddcdaa4410b00a467ee07111e444e87d649eda Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Mon, 7 Feb 2022 12:42:55 +0100 Subject: [PATCH 3/4] Add simple check with regex to decide which parameter to use This also adds a new function that calls the for loop that triggers the repolinter, with the proper parameter as an argument of the function Signed-off-by: Brend Smits --- bin/loop.sh | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/bin/loop.sh b/bin/loop.sh index 597cd77..3d1523a 100755 --- a/bin/loop.sh +++ b/bin/loop.sh @@ -1,12 +1,30 @@ #!/bin/bash [ -n "$1" ] && GITHUB_TOKEN=$1 || exit 1 -[ -n "$2" ] && RULESET_FILE_URL=$2 || exit 1 +[ -n "$2" ] && RULESET=$2 || exit 1 [ -n "$3" ] && TARGET_REPOS=$3 || exit 1 -for i in $(echo "$TARGET_REPOS" | sed "s/,/ /g"); do - export TARGET_REPO=$i - echo " checking: $i" - /app/repolinter/bin/repolinter.js lint \ - --rulesetUrl "$RULESET_FILE_URL" \ - -g https://x-access-token:"$GITHUB_TOKEN"@github.com/"${i}" -done +# Function that checks the target repositories with repolinter +# $1 is the ruleset argument, valid examples here are: +# 1. --rulesetFile /config/ruleset.json +# 2. --rulesetUrl https://gist.github.com/ruleset.json +function check_target_repos { + for i in $(echo "$TARGET_REPOS" | sed "s/,/ /g"); do + export TARGET_REPO=$i + echo "Checking repository: $i" + echo "Using ruleset parameter: $1" + /app/repolinter/bin/repolinter.js lint \ + "$1" \ + -g https://x-access-token:"$GITHUB_TOKEN"@github.com/"${i}" + done +} + +regex='(https?)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]' + +if [[ $RULESET =~ $regex ]] +then + echo "Link valid" + check_target_repos "--rulesetUrl ""$RULESET""" +else + echo "No link, assuming file path" + check_target_repos "--rulesetFile ""$RULESET""" +fi From 4101bde1c180dedd42a2d5e0484c3e9bb3cbc0f3 Mon Sep 17 00:00:00 2001 From: Brend Smits Date: Mon, 7 Feb 2022 12:46:25 +0100 Subject: [PATCH 4/4] Fix local continuous compliance action workflow Signed-off-by: Brend Smits --- .github/workflows/lint-using-action.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint-using-action.yml b/.github/workflows/lint-using-action.yml index 7168e0f..7db2342 100644 --- a/.github/workflows/lint-using-action.yml +++ b/.github/workflows/lint-using-action.yml @@ -14,9 +14,7 @@ jobs: steps: - name: Run Linter uses: philips-labs/continuous-compliance@main - env: - target_repos: ${{ github.event.inputs.target_repos }} - GITHUB_TOKEN: ${{ secrets.TEMP_TOKEN }} with: target_repos: ${{ github.event.inputs.target_repos }} - gh_token: ${{ secrets.TEMP_TOKEN }} \ No newline at end of file + gh_token: ${{ secrets.TEMP_TOKEN }} + ruleset: "https://gist.githubusercontent.com/Brend-Smits/385824d3422c0c06323bb1412cd5ba96/raw/08676dbf366a86641dd30fd5511b91afdee65adc/gistfile1.txt" \ No newline at end of file